@maschinenlesbar.org/ckan-cli 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -208,7 +208,7 @@ export class CkanClient {
208
208
  // A success:false envelope can come with HTTP 200, so it never passes the
209
209
  // engine's error-detail sanitising: strip terminal controls here too.
210
210
  const type = env.error?.__type;
211
- throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined, url });
211
+ throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(this.#engine.scrub(type))) : undefined, url });
212
212
  }
213
213
  // `{"success": true}` without a result would print nothing useful (and the CLI
214
214
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
@@ -131,19 +131,25 @@ export declare function isBidiControl(code: number): boolean;
131
131
  * `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
132
132
  * into an error message on stderr:
133
133
  *
134
- * - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
134
+ * - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
135
+ * then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
135
136
  * (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
136
137
  * or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
137
138
  * spoofing, title changes, OSC 52 clipboard writes).
138
- * - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
139
- * reorder the visible message.
139
+ * - Unicode format characters (General Category Cf) are dropped: the bidi controls
140
+ * (isBidiControl), so server text cannot reorder the visible message, and the
141
+ * invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
142
+ * two messages that look identical are identical.
140
143
  * - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
141
144
  * space and the ends are trimmed, so the text stays on one line and a server
142
145
  * cannot forge an `Error:` line of its own.
146
+ * - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
147
+ * regex replace: a hostile error body costs neither seconds nor the heap. A caller that
148
+ * scrubs credentials does so first, so a cut never splits one.
143
149
  *
144
150
  * The CLI's JSON output is escaped separately (`escapeControlChars` in
145
151
  * cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
146
- * Written as a char-code filter so no raw control byte appears in this source.
152
+ * Written with escapes so no raw control byte appears in this source.
147
153
  */
148
154
  export declare function sanitizeServerText(text: string): string;
149
155
  /**
@@ -200,6 +206,15 @@ export declare class RequestEngine {
200
206
  * messages built from server text.
201
207
  */
202
208
  scrub(text: string): string;
209
+ /**
210
+ * An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
211
+ * with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
212
+ * credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
213
+ * a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
214
+ * with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
215
+ * decodes, and stays as sent; one too deeply nested to write again is left out.
216
+ */
217
+ private scrubBody;
203
218
  /**
204
219
  * A failure as the `cause` of the error the engine raises: the original when its
205
220
  * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
@@ -223,6 +238,13 @@ export declare class RequestEngine {
223
238
  * becomes a rejection.
224
239
  */
225
240
  private callTransport;
241
+ /**
242
+ * The URL a redirect leads to as messages and errors show it: the Location with the base URL's
243
+ * credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
244
+ * spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
245
+ * left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
246
+ */
247
+ private shownTarget;
226
248
  /** Perform a request with Accept negotiation and transient-error retries. */
227
249
  request(method: string, path: string, options?: {
228
250
  query?: QueryParams;
@@ -5,7 +5,7 @@ import { constants } from "node:buffer";
5
5
  import { TextDecoder } from "node:util";
6
6
  import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
7
7
  import { buildQueryString } from "./query.js";
8
- import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
8
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, cutText, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, TextBuilder, } from "./errors.js";
9
9
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
10
10
  // quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
11
11
  export { quoteValue } from "./errors.js";
@@ -80,35 +80,57 @@ export function isBidiControl(code) {
80
80
  (code >= 0x202a && code <= 0x202e) ||
81
81
  (code >= 0x2066 && code <= 0x2069));
82
82
  }
83
+ /**
84
+ * The most server text (in UTF-16 units) `sanitizeServerText` reads; a longer text is cut first.
85
+ * Every message keeps far less of it (MAX_MESSAGE_VALUE_LENGTH, 500 characters), and cleaning all
86
+ * of a 16 MB error detail, one string node per character, exhausted a small heap.
87
+ */
88
+ const MAX_SERVER_TEXT = 16 * 1024;
89
+ /**
90
+ * A whole terminal escape sequence: a CSI (`ESC [` or the 8-bit U+009B, parameters,
91
+ * intermediates, final byte), a terminated control string (OSC/DCS/SOS/PM/APC, `ESC ]` …
92
+ * up to BEL or ST, or their 8-bit forms) or a short `ESC` sequence. Dropping only the
93
+ * control bytes left the printable rest behind (`ESC ] 0 ; title BEL` became `]0;title`).
94
+ * An unterminated control string is left to the per-character filter, so it cannot
95
+ * swallow the rest of the message. A control string's body ends at an ESC or at another
96
+ * introducer: with the introducers allowed inside it, an unterminated run of them was matched
97
+ * from each one to the end of the text, quadratic (100 000 × U+009D took 16 s). Written with
98
+ * escapes, so the source holds no control bytes.
99
+ */
100
+ const ESCAPE_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]|(?:\u001b[\]PX^_]|[\u0090\u0098\u009d-\u009f])[^\u0007\u001b\u0090\u0098\u009c-\u009f]*(?:\u0007|\u001b\\|\u009c)|\u001b[ -/]*[0-~]/g;
101
+ /**
102
+ * What `sanitizeServerText` drops once the escape sequences are gone: the C0 controls but the
103
+ * whitespace ones (TAB to CR), DEL, the C1 controls, and every format character (General
104
+ * Category Cf: bidi controls, zero-width characters, U+00AD, U+FEFF, …).
105
+ */
106
+ const DROPPED_CHAR = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f\p{Cf}]/gu;
83
107
  /**
84
108
  * Make a string that originates in an attacker-controlled response — the error
85
109
  * `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
86
110
  * into an error message on stderr:
87
111
  *
88
- * - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
112
+ * - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
113
+ * then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
89
114
  * (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
90
115
  * or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
91
116
  * spoofing, title changes, OSC 52 clipboard writes).
92
- * - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
93
- * reorder the visible message.
117
+ * - Unicode format characters (General Category Cf) are dropped: the bidi controls
118
+ * (isBidiControl), so server text cannot reorder the visible message, and the
119
+ * invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
120
+ * two messages that look identical are identical.
94
121
  * - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
95
122
  * space and the ends are trimmed, so the text stays on one line and a server
96
123
  * cannot forge an `Error:` line of its own.
124
+ * - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
125
+ * regex replace: a hostile error body costs neither seconds nor the heap. A caller that
126
+ * scrubs credentials does so first, so a cut never splits one.
97
127
  *
98
128
  * The CLI's JSON output is escaped separately (`escapeControlChars` in
99
129
  * cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
100
- * Written as a char-code filter so no raw control byte appears in this source.
130
+ * Written with escapes so no raw control byte appears in this source.
101
131
  */
102
132
  export function sanitizeServerText(text) {
103
- let out = "";
104
- for (const ch of text) {
105
- const n = ch.codePointAt(0) ?? 0;
106
- const whitespaceControl = n >= 0x09 && n <= 0x0d;
107
- if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
108
- continue;
109
- out += ch;
110
- }
111
- return out.replace(/\s+/g, " ").trim();
133
+ return cutText(text, MAX_SERVER_TEXT).replace(ESCAPE_SEQUENCE, "").replace(DROPPED_CHAR, "").replace(/\s+/g, " ").trim();
112
134
  }
113
135
  /**
114
136
  * Check a base URL (baseUrlProblem: not blank, no whitespace or control
@@ -246,6 +268,8 @@ export class RequestEngine {
246
268
  // JSON.stringify of a client never show it, so a password in the base URL can't be
247
269
  // logged by accident. Messages use redactUrl.
248
270
  #baseUrl;
271
+ /** `#baseUrl` without its userinfo: what every request URL starts with. */
272
+ #requestBase;
249
273
  /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
250
274
  #credentials;
251
275
  /**
@@ -272,8 +296,18 @@ export class RequestEngine {
272
296
  // consumer that injects a custom transport would otherwise get no gating at
273
297
  // all, and could be steered to a non-http(s) scheme. Only undefined selects
274
298
  // the default.
275
- this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
276
- this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
299
+ const baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
300
+ // Requests go to the base URL as WHATWG parses it, the reading every check of it makes (the
301
+ // host, the cleartext warning, the Authorization header), never to a text-based reading of
302
+ // the string: a transport that reads the URL as text reaches the same host.
303
+ const parsed = new URL(baseUrl);
304
+ this.#baseUrl = parsed.href.replace(/\/+$/, "");
305
+ parsed.username = "";
306
+ parsed.password = "";
307
+ this.#requestBase = parsed.href.replace(/\/+$/, "");
308
+ // The userinfo as typed and as sent (WHATWG percent-encodes some of its characters).
309
+ const userinfos = [...new Set([...credentialsIn(baseUrl), ...credentialsIn(this.#baseUrl)])];
310
+ this.#credentials = userinfos.flatMap((raw) => {
277
311
  try {
278
312
  return [raw, decodeURIComponent(raw)];
279
313
  }
@@ -281,8 +315,7 @@ export class RequestEngine {
281
315
  return [raw];
282
316
  }
283
317
  });
284
- this.#echoed = credentialsIn(this.#baseUrl)
285
- .flatMap(echoedCredentialForms)
318
+ this.#echoed = [...new Set(userinfos.flatMap(echoedCredentialForms).flatMap(urlSpellings))]
286
319
  .sort((a, b) => b.length - a.length);
287
320
  this.transport = functionOption("transport", options.transport, nodeHttpTransport);
288
321
  // Only undefined selects the default; a blank or unsendable value is refused.
@@ -338,6 +371,34 @@ export class RequestEngine {
338
371
  scrub(text) {
339
372
  return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
340
373
  }
374
+ /**
375
+ * An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
376
+ * with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
377
+ * credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
378
+ * a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
379
+ * with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
380
+ * decodes, and stays as sent; one too deeply nested to write again is left out.
381
+ */
382
+ scrubBody(scrubbed) {
383
+ if (this.#echoed.length === 0 || !scrubbed.includes("\\"))
384
+ return scrubbed;
385
+ let decoded;
386
+ try {
387
+ decoded = JSON.parse(scrubbed);
388
+ }
389
+ catch {
390
+ return scrubbed;
391
+ }
392
+ let rewritten;
393
+ try {
394
+ rewritten = JSON.stringify(decoded);
395
+ }
396
+ catch {
397
+ return "";
398
+ }
399
+ const spelled = this.#echoed.map((form) => JSON.stringify(form).slice(1, -1));
400
+ return spelled.some((form) => rewritten.includes(form)) ? redactSecrets(rewritten, spelled) : scrubbed;
401
+ }
341
402
  /**
342
403
  * A failure as the `cause` of the error the engine raises: the original when its
343
404
  * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
@@ -375,7 +436,7 @@ export class RequestEngine {
375
436
  composeUrl(path, query, withUserinfo) {
376
437
  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
377
438
  const qs = query ? buildQueryString(query) : "";
378
- const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
439
+ const base = withUserinfo ? this.#baseUrl : this.#requestBase;
379
440
  return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
380
441
  }
381
442
  /**
@@ -405,6 +466,24 @@ export class RequestEngine {
405
466
  clearTimeout(timer);
406
467
  }
407
468
  }
469
+ /**
470
+ * The URL a redirect leads to as messages and errors show it: the Location with the base URL's
471
+ * credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
472
+ * spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
473
+ * left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
474
+ */
475
+ shownTarget(location, shown) {
476
+ const sent = this.scrub(location);
477
+ try {
478
+ const target = new URL(sent, shown);
479
+ target.username = "";
480
+ target.password = "";
481
+ return target.href;
482
+ }
483
+ catch {
484
+ return sent;
485
+ }
486
+ }
408
487
  /** Perform a request with Accept negotiation and transient-error retries. */
409
488
  async request(method, path, options = { accept: "application/json" }) {
410
489
  // The transport never sees the base URL's userinfo: the engine sends it as an
@@ -412,6 +491,9 @@ export class RequestEngine {
412
491
  // absolute) keeps it and one to another origin or scheme drops it. A transport
413
492
  // such as fetch also refuses a URL with credentials outright.
414
493
  let url = this.composeUrl(path, options.query, false);
494
+ // The URL as messages, errors and `RawResponse.url` show it: after a redirect, the Location
495
+ // with the base URL's credentials scrubbed (shownTarget); `url` is what is requested.
496
+ let shown = url;
415
497
  let headers = {
416
498
  Accept: options.accept,
417
499
  "User-Agent": this.userAgent,
@@ -445,7 +527,7 @@ export class RequestEngine {
445
527
  if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
446
528
  throw cause;
447
529
  const reason = cause instanceof Error ? cause.message : String(cause);
448
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
530
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
449
531
  cause: this.scrubCause(cause),
450
532
  });
451
533
  }
@@ -453,14 +535,14 @@ export class RequestEngine {
453
535
  // would otherwise surface below as a raw TypeError, outside the CkanError contract.
454
536
  const invalid = responseProblem(response);
455
537
  if (invalid !== undefined) {
456
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
538
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport returned an invalid response (${invalid}).`);
457
539
  }
458
540
  // A transport must not follow redirects itself (`redirect: "manual"`): one that
459
541
  // did (fetch's default) may have carried the Authorization header to another
460
542
  // host, and the answer is not the one asked for. Reject it when it says so (`url`).
461
543
  const finalUrl = response.url;
462
544
  if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
463
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
545
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport followed a redirect to another origin ` +
464
546
  `(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
465
547
  `(HttpRequest.redirect is "manual").`);
466
548
  }
@@ -471,7 +553,7 @@ export class RequestEngine {
471
553
  // The size cap holds whatever the transport did: the default one aborts early, a
472
554
  // custom one may have read everything.
473
555
  if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
474
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
556
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
475
557
  }
476
558
  const retryable = status === 429 || status === 503;
477
559
  // A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
@@ -484,7 +566,7 @@ export class RequestEngine {
484
566
  // for less: `Retry-After: 0` or a date in the past turned the retries into a
485
567
  // zero-delay burst against a server that had just asked for less load.
486
568
  const backoff = this.retryDelayMs * attempt;
487
- await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
569
+ await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), shown, status);
488
570
  continue;
489
571
  }
490
572
  // Follow redirects, resolving the Location relative to the current URL.
@@ -500,7 +582,7 @@ export class RequestEngine {
500
582
  // never reaches a transport (a custom one may not check the scheme).
501
583
  if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
502
584
  throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
503
- `for ${method} ${cutForMessage(redactUrl(url))}`);
585
+ `for ${method} ${cutForMessage(redactUrl(shown))}`);
504
586
  }
505
587
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
506
588
  // port, so an https->http downgrade counts), keep only the engine's own
@@ -521,6 +603,7 @@ export class RequestEngine {
521
603
  headers = engineHeadersOnly(headers);
522
604
  }
523
605
  url = nextUrl.toString();
606
+ shown = this.shownTarget(location, shown);
524
607
  redirects += 1;
525
608
  continue;
526
609
  }
@@ -535,16 +618,16 @@ export class RequestEngine {
535
618
  const detail = target !== undefined
536
619
  ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
537
620
  : location
538
- ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
621
+ ? `redirect to ${cutForMessage(sanitizeServerText(this.scrub(location)))} not followed`
539
622
  : "redirect not followed (no Location header)";
540
- throw new CkanApiError({ status, url, method, body: this.scrub(errorBodyText(body)), detail });
623
+ throw new CkanApiError({ status, url: shown, method, body: this.scrubBody(this.scrub(errorBodyText(body))), detail });
541
624
  }
542
- throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
625
+ throw this.toApiError(method, shown, status, body, status === 401 || status === 403 ? dropped : undefined, {
543
626
  retries: attempt,
544
627
  ...(tooLong ? { retryAfterMs: retryAfter } : {}),
545
628
  });
546
629
  }
547
- return { data: body, contentType, status, url };
630
+ return { data: body, contentType, status, url: shown };
548
631
  }
549
632
  }
550
633
  /** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
@@ -627,28 +710,24 @@ export class RequestEngine {
627
710
  catch {
628
711
  // Non-JSON error body; leave detail undefined.
629
712
  }
630
- // `detail` came from the response body; strip control characters so a hostile
713
+ // `detail` came from the response body, decoded: scrubbed again (a JSON escape hid the
714
+ // credential from the scrub above), then stripped of control characters so a hostile
631
715
  // endpoint cannot inject terminal escape sequences via the stderr error message.
632
716
  if (detail !== undefined)
633
- detail = sanitizeServerText(detail);
717
+ detail = sanitizeServerText(this.scrub(detail));
634
718
  if (hint !== undefined)
635
719
  detail = detail === undefined ? hint : `${detail}; ${hint}`;
636
720
  return new CkanApiError({
637
721
  status,
638
722
  url,
639
723
  method,
640
- body: text,
724
+ body: this.scrubBody(text),
641
725
  detail,
642
726
  retries: retry.retries,
643
727
  ...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
644
728
  });
645
729
  }
646
730
  }
647
- /** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
648
- function withoutUserinfo(url) {
649
- const [userinfo] = credentialsIn(url);
650
- return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
651
- }
652
731
  /**
653
732
  * The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
654
733
  * both percent-decoded, as Node's own http client builds it), or undefined without
@@ -770,7 +849,7 @@ function escapeAt(json, at) {
770
849
  function wellFormedEscapes(json) {
771
850
  if (!SURROGATE_ESCAPE.test(json))
772
851
  return { json, replaced: 0 };
773
- let out = "";
852
+ const out = new TextBuilder();
774
853
  let copied = 0;
775
854
  let replaced = 0;
776
855
  for (let i = 0; i < json.length; i++) {
@@ -786,16 +865,32 @@ function wellFormedEscapes(json) {
786
865
  i += 11;
787
866
  continue;
788
867
  }
789
- out += json.slice(copied, i) + "\\ufffd";
868
+ out.push(json.slice(copied, i), "\\ufffd");
790
869
  copied = i + 6;
791
870
  replaced++;
792
871
  i += 5;
793
872
  }
794
- return replaced === 0 ? { json, replaced } : { json: out + json.slice(copied), replaced };
873
+ if (replaced === 0)
874
+ return { json, replaced };
875
+ out.push(json.slice(copied));
876
+ return { json: out.toString(), replaced };
795
877
  }
796
878
  function isLowSurrogate(code) {
797
879
  return code >= 0xdc00 && code <= 0xdfff;
798
880
  }
881
+ /**
882
+ * `form` and its spelling in a URL (`encodeURIComponent`): a server that puts a credential into
883
+ * a URL, a redirect's Location, percent-encodes it. A form that has no such spelling (a lone
884
+ * surrogate) is kept as it is.
885
+ */
886
+ function urlSpellings(form) {
887
+ try {
888
+ return [form, encodeURIComponent(form)];
889
+ }
890
+ catch {
891
+ return [form];
892
+ }
893
+ }
799
894
  /** Resolve a Location header against the current URL; undefined if missing or malformed. */
800
895
  function resolveLocation(location, base) {
801
896
  if (location === undefined || location === "")
@@ -833,9 +928,17 @@ function exactAtTheEdges(literal, value) {
833
928
  /** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
834
929
  function decimalOf(text) {
835
930
  const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
836
- const all = (whole + fraction).replace(/^0+/, "");
837
- const digits = all.replace(/0+$/, "");
838
- return { negative: sign === "-", digits, exponent: Number(exp) - fraction.length + (all.length - digits.length) };
931
+ // The zeros are counted with index loops: `/0+$/` tried a match at every zero of a run that
932
+ // doesn't end the text, quadratic in the run, and one `1.<64 000 zeros>1e300` in an answer held
933
+ // the CLI for seconds, past --timeout (the scan runs once the answer is in).
934
+ const all = whole + fraction;
935
+ let start = 0;
936
+ while (start < all.length && all.charCodeAt(start) === 0x30)
937
+ start++;
938
+ let end = all.length;
939
+ while (end > start && all.charCodeAt(end - 1) === 0x30)
940
+ end--;
941
+ return { negative: sign === "-", digits: all.slice(start, end), exponent: Number(exp) - fraction.length + (all.length - end) };
839
942
  }
840
943
  /** A JSON number token as the grammar allows it, matched at a given position (sticky). */
841
944
  const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
@@ -902,7 +1005,8 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
902
1005
  const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
903
1006
  if (literal === undefined)
904
1007
  continue;
905
- out = (out ?? "") + json.slice(copied, at) + '"' + literal + '"';
1008
+ out ??= new TextBuilder();
1009
+ out.push(json.slice(copied, at), '"', literal, '"');
906
1010
  copied = at + literal.length;
907
1011
  i = copied - 1;
908
1012
  }
@@ -920,7 +1024,10 @@ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
920
1024
  i += literal.length - 1;
921
1025
  }
922
1026
  }
923
- return { json: out === undefined ? json : out + json.slice(copied), inexact };
1027
+ if (out === undefined)
1028
+ return { json, inexact };
1029
+ out.push(json.slice(copied));
1030
+ return { json: out.toString(), inexact };
924
1031
  }
925
1032
  /** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
926
1033
  function decodedName(quoted) {
@@ -9,20 +9,60 @@ export declare function redactUrl(url: string): string;
9
9
  /**
10
10
  * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
11
11
  * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
12
- * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
12
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
13
+ * slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
13
14
  * search text or a User-Agent as often as a credential, and the base URL always has a
14
15
  * scheme. It works on URLs that don't parse too: the userinfo is everything between
15
16
  * `://` and the last `@` before the host. Used to redact those exact strings from text
16
17
  * that echoes the value (usage errors, help), whatever characters the password contains.
17
18
  */
18
19
  export declare function credentialsIn(value: string): string[];
20
+ /**
21
+ * The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
22
+ * typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
23
+ * (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
24
+ * (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
25
+ * first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
26
+ * doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
27
+ * it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
28
+ * other text a bare `a/b@c` is no credential.
29
+ */
30
+ export declare function typedUrlCredentials(value: string): string[];
31
+ /**
32
+ * `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
33
+ * becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
34
+ * so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
35
+ * whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
36
+ * `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
37
+ * and the search goes on right after each `://`, not after the authority it read: a URL glued
38
+ * onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
39
+ * authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
40
+ * and every character is read a bounded number of times: by the authority loop and `indexOf`
41
+ * forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
42
+ * word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
43
+ */
44
+ export declare function redactUrlUserinfo(text: string): string;
45
+ /**
46
+ * A text put together from many pieces (slices of a server answer and their replacements),
47
+ * joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
48
+ * not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
49
+ * piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
50
+ * either way.
51
+ */
52
+ export declare class TextBuilder {
53
+ #private;
54
+ push(...pieces: string[]): void;
55
+ toString(): string;
56
+ }
19
57
  /**
20
58
  * The forms in which a server may echo the credentials of a userinfo (`user:password`,
21
- * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
22
- * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
23
- * `user:password` itself, and the password alone when it is at least 4 characters long.
24
- * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
25
- * are replaced as exact strings (`redactSecrets`).
59
+ * as `credentialsIn` returns it) back in an error body, in this order: the
60
+ * `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
61
+ * sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
62
+ * characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
63
+ * `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
64
+ * for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
65
+ * strings (`redactSecrets`).
26
66
  */
27
67
  export declare function echoedCredentialForms(userinfo: string): string[];
28
68
  /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
@@ -83,6 +123,12 @@ export declare class CkanApiError extends CkanError {
83
123
  readonly detail: string | undefined;
84
124
  readonly url: string;
85
125
  readonly method: string;
126
+ /**
127
+ * The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
128
+ * whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
129
+ * written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
130
+ * for such a body nested too deeply to write again.
131
+ */
86
132
  readonly body: string;
87
133
  /** How many times the engine retried the request before giving up (0 when it did not). */
88
134
  readonly retries: number;
@@ -27,14 +27,17 @@ export function redactUrl(url) {
27
27
  /**
28
28
  * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
29
29
  * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
30
- * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
30
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
31
+ * slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
31
32
  * search text or a User-Agent as often as a credential, and the base URL always has a
32
33
  * scheme. It works on URLs that don't parse too: the userinfo is everything between
33
34
  * `://` and the last `@` before the host. Used to redact those exact strings from text
34
35
  * that echoes the value (usage errors, help), whatever characters the password contains.
35
36
  */
36
37
  export function credentialsIn(value) {
37
- const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
+ // WHATWG skips any number of "/" and "\" after a special scheme (`http:///u:pw@h`,
39
+ // `https:\\u:pw@h` and `http:u:pw@h` all send `u:pw`), so its userinfo starts after them.
40
+ const scheme = SPECIAL_SCHEME_START.exec(value) ?? /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
41
  if (scheme === null)
39
42
  return [];
40
43
  const rest = value.slice(scheme[0].length);
@@ -47,23 +50,138 @@ export function credentialsIn(value) {
47
50
  // Doesn't parse: the password may hold "/", "?", "#" or spaces.
48
51
  }
49
52
  // In a URL that parses, the userinfo ends at the last "@" of the authority (before the
50
- // first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
51
- const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
53
+ // first "/", "?" or "#", and for a special scheme such as http(s) a "\", where WHATWG ends
54
+ // it too: `https://u:pw@a.example\@b.example` sends `u:pw` to a.example); in one that doesn't,
55
+ // at the last "@" of the value.
56
+ const stop = SPECIAL_SCHEME.test(scheme[0]) ? /[/?#\\]|$/ : /[/?#]|$/;
57
+ const authority = parses ? rest.slice(0, rest.search(stop)) : rest;
52
58
  const end = authority.lastIndexOf("@");
53
59
  return end > 0 ? [rest.slice(0, end)] : [];
54
60
  }
61
+ /**
62
+ * The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
63
+ * typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
64
+ * (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
65
+ * (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
66
+ * first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
67
+ * doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
68
+ * it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
69
+ * other text a bare `a/b@c` is no credential.
70
+ */
71
+ export function typedUrlCredentials(value) {
72
+ if (value === "" || credentialsIn(value).length > 0 || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
73
+ return credentialsIn(value);
74
+ const prefixed = credentialsIn(`http://${value}`);
75
+ if (prefixed.length > 0)
76
+ return prefixed;
77
+ const slashes = /[/\\]+/.exec(value);
78
+ const at = value.lastIndexOf("@");
79
+ if (slashes === null || slashes.index + slashes[0].length >= at)
80
+ return [];
81
+ return [value.slice(slashes.index + slashes[0].length, at)];
82
+ }
83
+ /** The schemes WHATWG parses as special (`ftp`, `file`, `http(s)`, `ws(s)`): their authority also ends at a backslash. */
84
+ const SPECIAL_SCHEME = /^(?:ftp|file|https?|wss?):[/\\]*$/i;
85
+ /** A special scheme at the start of a value, with the slashes and backslashes WHATWG skips after it. */
86
+ const SPECIAL_SCHEME_START = /^(?:ftp|file|https?|wss?):[/\\]*/i;
87
+ /** A scheme character (RFC 3986: letters, digits, `+`, `.`, `-`). */
88
+ function isSchemeChar(code) {
89
+ return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39) || code === 0x2b || code === 0x2e || code === 0x2d;
90
+ }
91
+ /** An ASCII letter, with which a scheme starts. */
92
+ function isLetter(code) {
93
+ return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);
94
+ }
95
+ /**
96
+ * Where the scheme of the `://` at `at` starts: the first letter of the run of scheme characters
97
+ * before it, looking back no further than `from`; -1 when there is none (`1://`, ` ://`). A run
98
+ * led by other scheme characters (`-https://`) starts at its first letter.
99
+ */
100
+ function schemeStart(text, at, from) {
101
+ let start = at;
102
+ while (start > from && isSchemeChar(text.charCodeAt(start - 1)))
103
+ start--;
104
+ while (start < at && !isLetter(text.charCodeAt(start)))
105
+ start++;
106
+ return start < at ? start : -1;
107
+ }
108
+ /**
109
+ * `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
110
+ * becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
111
+ * so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
112
+ * whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
113
+ * `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
114
+ * and the search goes on right after each `://`, not after the authority it read: a URL glued
115
+ * onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
116
+ * authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
117
+ * and every character is read a bounded number of times: by the authority loop and `indexOf`
118
+ * forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
119
+ * word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
120
+ */
121
+ export function redactUrlUserinfo(text) {
122
+ const parts = new TextBuilder();
123
+ let copied = 0;
124
+ let from = 0;
125
+ for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", from)) {
126
+ const begin = at + 3;
127
+ let end = begin;
128
+ let lastAt = -1;
129
+ for (; end < text.length; end++) {
130
+ const c = text.charCodeAt(end);
131
+ if (c === 0x2f || c === 0x3f || c === 0x23 || c === 0x27 || /\s/.test(text[end]))
132
+ break;
133
+ if (c === 0x40)
134
+ lastAt = end;
135
+ }
136
+ if (lastAt !== -1 && schemeStart(text, at, from) !== -1) {
137
+ parts.push(text.slice(copied, begin), "***");
138
+ copied = lastAt;
139
+ }
140
+ from = begin;
141
+ }
142
+ if (copied === 0)
143
+ return text;
144
+ parts.push(text.slice(copied));
145
+ return parts.toString();
146
+ }
147
+ /** How many pieces a TextBuilder joins at a time. */
148
+ const BUILDER_BATCH = 16_384;
149
+ /**
150
+ * A text put together from many pieces (slices of a server answer and their replacements),
151
+ * joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
152
+ * not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
153
+ * piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
154
+ * either way.
155
+ */
156
+ export class TextBuilder {
157
+ #done = [];
158
+ #batch = [];
159
+ push(...pieces) {
160
+ for (const piece of pieces)
161
+ this.#batch.push(piece);
162
+ if (this.#batch.length >= BUILDER_BATCH) {
163
+ this.#done.push(this.#batch.join(""));
164
+ this.#batch = [];
165
+ }
166
+ }
167
+ toString() {
168
+ this.#done.push(this.#batch.join(""));
169
+ this.#batch = [];
170
+ return this.#done.join("");
171
+ }
172
+ }
55
173
  /**
56
174
  * The forms in which a server may echo the credentials of a userinfo (`user:password`,
57
- * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
58
- * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
59
- * `user:password` itself, and the password alone when it is at least 4 characters long.
60
- * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
61
- * are replaced as exact strings (`redactSecrets`).
175
+ * as `credentialsIn` returns it) back in an error body, in this order: the
176
+ * `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
177
+ * sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
178
+ * characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
179
+ * `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
180
+ * for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
181
+ * strings (`redactSecrets`).
62
182
  */
63
183
  export function echoedCredentialForms(userinfo) {
64
184
  const colon = userinfo.indexOf(":");
65
- if (colon < 0)
66
- return [];
67
185
  const decode = (part) => {
68
186
  try {
69
187
  return decodeURIComponent(part);
@@ -72,14 +190,15 @@ export function echoedCredentialForms(userinfo) {
72
190
  return part;
73
191
  }
74
192
  };
75
- const user = decode(userinfo.slice(0, colon));
76
- const password = decode(userinfo.slice(colon + 1));
77
- if (password === "")
193
+ const user = decode(colon < 0 ? userinfo : userinfo.slice(0, colon));
194
+ const password = colon < 0 ? "" : decode(userinfo.slice(colon + 1));
195
+ const secret = password === "" ? user : password;
196
+ if (secret === "")
78
197
  return [];
79
198
  const pair = `${user}:${password}`;
80
199
  const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
81
- if (password.length >= 4)
82
- forms.push(password);
200
+ if (secret.length >= 4)
201
+ forms.push(secret);
83
202
  return forms;
84
203
  }
85
204
  /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
@@ -186,6 +305,12 @@ export class CkanApiError extends CkanError {
186
305
  detail;
187
306
  url;
188
307
  method;
308
+ /**
309
+ * The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
310
+ * whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
311
+ * written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
312
+ * for such a body nested too deeply to write again.
313
+ */
189
314
  body;
190
315
  /** How many times the engine retried the request before giving up (0 when it did not). */
191
316
  retries;
@@ -291,8 +416,21 @@ export function describeCkanError(error) {
291
416
  * quotes and newlines escaped several levels deep.
292
417
  */
293
418
  function solrReason(message) {
294
- const match = /\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/.exec(message);
419
+ // Only a window after the first "[Reason: " is read: the pattern's lazy group, tried against
420
+ // `\\+n` at every step, is quadratic on a run of backslashes, and the message is uncut server
421
+ // text (80 KB took 2.4 s, past --timeout, and it grows with the square). The window is twice what a
422
+ // message shows; a reason that runs past it ends at the window's end.
423
+ const at = message.indexOf(REASON);
424
+ if (at === -1)
425
+ return undefined;
426
+ const end = at + REASON.length + 2 * MAX_MESSAGE_VALUE_LENGTH;
427
+ const window = message.slice(at, end);
428
+ const pattern = end < message.length ? SOLR_REASON_IN_CUT_WINDOW : SOLR_REASON;
429
+ const match = pattern.exec(window);
295
430
  if (!match?.[1])
296
431
  return undefined;
297
432
  return match[1].replace(/\\+(['"])/g, "$1").trim();
298
433
  }
434
+ const REASON = "[Reason: ";
435
+ const SOLR_REASON = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/;
436
+ const SOLR_REASON_IN_CUT_WINDOW = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\]|$)/;
@@ -48,6 +48,7 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
48
48
  /**
49
49
  * Default transport. Resolves with the raw response (including non-2xx) — status
50
50
  * interpretation is the client's job. Rejects only on transport-level failures
51
- * (connection errors, timeouts, malformed URLs).
51
+ * (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
52
+ * number of chunks a server sends it in: small chunks are joined as they arrive.
52
53
  */
53
54
  export declare const nodeHttpTransport: Transport;
@@ -7,7 +7,7 @@
7
7
  // `http.createServer` in the test-suite.
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
- import { CkanNetworkError, redactUrl } from "./errors.js";
10
+ import { CkanNetworkError, redactUrl, credentialsIn, redactCredentials, typedUrlCredentials } from "./errors.js";
11
11
  /** The message for a body over the size cap, naming the option on both sides. */
12
12
  export function sizeLimitMessage(maxBytes) {
13
13
  return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
@@ -17,10 +17,23 @@ export function sizeLimitMessage(maxBytes) {
17
17
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
18
18
  */
19
19
  export const MAX_TIMEOUT_MS = 2_147_483_647;
20
+ /** The most received chunks the default transport keeps unjoined. */
21
+ const COALESCE_CHUNKS = 1024;
22
+ /** The bytes of unjoined chunks at which the default transport joins them into one Buffer. */
23
+ const COALESCE_BYTES = 64 * 1024;
24
+ /**
25
+ * `url` for a message, its userinfo redacted; one typed without its scheme (`user:pw@host`,
26
+ * `TOKEN:@host`) or with a mistyped one (`http//user:pw@host`) is read as a URL all the same
27
+ * (`typedUrlCredentials`), since `redactUrl` finds a userinfo only after a scheme.
28
+ */
29
+ function shownUrl(url) {
30
+ return credentialsIn(url).length > 0 ? redactUrl(url) : redactCredentials(url, typedUrlCredentials(url));
31
+ }
20
32
  /**
21
33
  * Default transport. Resolves with the raw response (including non-2xx) — status
22
34
  * interpretation is the client's job. Rejects only on transport-level failures
23
- * (connection errors, timeouts, malformed URLs).
35
+ * (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
36
+ * number of chunks a server sends it in: small chunks are joined as they arrive.
24
37
  */
25
38
  export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
26
39
  let url;
@@ -28,14 +41,17 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
28
41
  url = new URL(request.url);
29
42
  }
30
43
  catch {
31
- reject(new CkanNetworkError(`Invalid URL: ${redactUrl(request.url)}`));
44
+ reject(new CkanNetworkError(`Invalid URL: ${shownUrl(request.url)}`));
32
45
  return;
33
46
  }
34
47
  // Only http/https are supported. Reject anything else up front with a clear,
35
48
  // typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
36
49
  // (and so this never reaches the file:/ftp:/etc. drivers).
37
50
  if (url.protocol !== "http:" && url.protocol !== "https:") {
38
- reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${redactUrl(request.url)}`));
51
+ // `user:password@host` without a scheme parses with the user name as the scheme: name a
52
+ // scheme only when the value spells one (`<scheme>://`).
53
+ const named = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(request.url) ? ` "${url.protocol}"` : "";
54
+ reject(new CkanNetworkError(`Unsupported protocol${named} in URL: ${shownUrl(request.url)}`));
39
55
  return;
40
56
  }
41
57
  const isHttps = url.protocol === "https:";
@@ -62,7 +78,12 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
62
78
  method: request.method,
63
79
  headers: request.headers,
64
80
  }, (res) => {
81
+ // Small chunks are joined as they arrive (`pending`, at most COALESCE_CHUNKS of them
82
+ // or COALESCE_BYTES), so the Buffers kept follow the bytes, not the number of chunks
83
+ // a server chooses to send: a Buffer per one-byte chunk held ~210 bytes of heap each.
65
84
  const chunks = [];
85
+ let pending = [];
86
+ let pendingBytes = 0;
66
87
  let received = 0;
67
88
  let aborted = false;
68
89
  res.on("data", (chunk) => {
@@ -75,7 +96,13 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
75
96
  settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
76
97
  return;
77
98
  }
78
- chunks.push(chunk);
99
+ pending.push(chunk);
100
+ pendingBytes += chunk.length;
101
+ if (pending.length >= COALESCE_CHUNKS || pendingBytes >= COALESCE_BYTES) {
102
+ chunks.push(pending.length === 1 ? chunk : Buffer.concat(pending, pendingBytes));
103
+ pending = [];
104
+ pendingBytes = 0;
105
+ }
79
106
  });
80
107
  res.on("end", () => {
81
108
  if (aborted)
@@ -83,7 +110,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
83
110
  settleResolve({
84
111
  status: res.statusCode ?? 0,
85
112
  headers: res.headers,
86
- body: Buffer.concat(chunks),
113
+ body: Buffer.concat([...chunks, ...pending]),
87
114
  });
88
115
  });
89
116
  res.on("error", (err) => {
@@ -117,13 +117,17 @@ export declare function headerValueProblem(value: string): string | undefined;
117
117
  *
118
118
  * - not blank;
119
119
  * - no surrounding or embedded whitespace and no control characters: `new URL`
120
- * trims and drops tab/newline silently, but the client glues the value into
121
- * every request URL as given, so a trailing space would request another path
122
- * (`/ckan%20/api/3/...`) or reach a custom transport raw;
120
+ * trims and drops tab/newline silently, so the URL requested would not be the one
121
+ * typed (a trailing space would request another path, `/ckan%20/api/3/...`);
123
122
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
124
- * transport);
123
+ * transport); another scheme is named only when the value spells `<scheme>://`, since
124
+ * `user:password@host` parses with the user name as its scheme;
125
125
  * - no query string or fragment: the API path is appended, so it would land in
126
126
  * front of `/api/3/action`;
127
+ * - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
128
+ * `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
129
+ * any slashes after the scheme, so text-based code would read another host or no userinfo
130
+ * (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
127
131
  * - a `%` in the user name or password must start a valid escape (`%25` for a
128
132
  * literal one): the engine decodes the userinfo for the Authorization header.
129
133
  *
@@ -278,13 +278,17 @@ export function headerValueProblem(value) {
278
278
  *
279
279
  * - not blank;
280
280
  * - no surrounding or embedded whitespace and no control characters: `new URL`
281
- * trims and drops tab/newline silently, but the client glues the value into
282
- * every request URL as given, so a trailing space would request another path
283
- * (`/ckan%20/api/3/...`) or reach a custom transport raw;
281
+ * trims and drops tab/newline silently, so the URL requested would not be the one
282
+ * typed (a trailing space would request another path, `/ckan%20/api/3/...`);
284
283
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
285
- * transport);
284
+ * transport); another scheme is named only when the value spells `<scheme>://`, since
285
+ * `user:password@host` parses with the user name as its scheme;
286
286
  * - no query string or fragment: the API path is appended, so it would land in
287
287
  * front of `/api/3/action`;
288
+ * - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
289
+ * `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
290
+ * any slashes after the scheme, so text-based code would read another host or no userinfo
291
+ * (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
288
292
  * - a `%` in the user name or password must start a valid escape (`%25` for a
289
293
  * literal one): the engine decodes the userinfo for the Authorization header.
290
294
  *
@@ -311,10 +315,23 @@ export function baseUrlProblem(value) {
311
315
  return "Expected an absolute http(s) URL.";
312
316
  }
313
317
  if (url.protocol !== "http:" && url.protocol !== "https:") {
318
+ // `user:password@host` without a scheme parses as a URL whose scheme is the user name
319
+ // ("t0kens3cret:"), lower-cased, which no exact-string redaction can match: name a scheme
320
+ // only when the value spells one, `<scheme>://`.
321
+ if (!/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
322
+ return "Expected an absolute http(s) URL.";
314
323
  return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
315
324
  }
316
325
  if (/[?#]/.test(value))
317
326
  return "Expected a site URL without a query string or fragment.";
327
+ // WHATWG reads a "\" in an http(s) URL as "/", so it ends the host, and it skips any number of
328
+ // slashes after "http:": `http://u:pw@a.example\@b.example` is a.example to the parser and
329
+ // was b.example to text-based code, and `http:u:pw@host` hid the userinfo from the redaction.
330
+ // Every reader of the base URL has to agree where its host is.
331
+ if (value.includes("\\"))
332
+ return 'A base URL cannot contain a backslash (\\); write "/" instead.';
333
+ if (!/^https?:\/\/[^/]/i.test(value))
334
+ return 'Expected an absolute http(s) URL ("https://" and the host).';
318
335
  // The engine decodes the userinfo into the Authorization header; a "%" that isn't an
319
336
  // escape would fail there ("URI malformed") at request time. Reject it here.
320
337
  for (const part of [url.username, url.password]) {
@@ -1,13 +1,6 @@
1
1
  import { type Command } from "commander";
2
2
  import { type CliDeps } from "./io.js";
3
3
  import { type Logger } from "./log.js";
4
- /**
5
- * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
6
- * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
7
- * covers a URL that does not parse; a fallback behind the exact strings of
8
- * `withRedactedOutput`.
9
- */
10
- export declare function redactUserinfo(text: string): string;
11
4
  /** The secrets of a run, and the two ways they are replaced. */
12
5
  export interface Redaction {
13
6
  /**
@@ -6,7 +6,7 @@ import { buildProgram, defaultDeps } from "./program.js";
6
6
  import { logOf } from "./io.js";
7
7
  import { OutputError } from "./output.js";
8
8
  import { createLogger, logFormatFromArgv } from "./log.js";
9
- import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "@maschinenlesbar.org/ckan-client/internal/errors";
9
+ import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, redactUrlUserinfo, typedUrlCredentials, } from "@maschinenlesbar.org/ckan-client/internal/errors";
10
10
  /** The exit code of a usage error: commander's own for a rejected option value. */
11
11
  const USAGE_EXIT = 1;
12
12
  /**
@@ -107,15 +107,6 @@ function escapeCommanderError(message) {
107
107
  return escapeTerminalText(message);
108
108
  return escapeTerminalText(message.slice(0, hint.index)) + hint[0];
109
109
  }
110
- /**
111
- * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
112
- * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
113
- * covers a URL that does not parse; a fallback behind the exact strings of
114
- * `withRedactedOutput`.
115
- */
116
- export function redactUserinfo(text) {
117
- return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
118
- }
119
110
  /**
120
111
  * The options whose value is a base URL (`--portal` takes a portal's URL as well as its
121
112
  * id): a `user:password@host` given there without its scheme is still a credential, like
@@ -182,14 +173,16 @@ export function redactionFor(argv, env) {
182
173
  });
183
174
  // A base URL typed without its scheme is read as if it had one (anywhere else a bare
184
175
  // `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
185
- const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
186
- const sources = [...values.flatMap(urlsWithin), ...baseUrls];
176
+ // Read as a URL however its scheme is typed: missing, special without "//", or mistyped
177
+ // (`http//u:pw@h`, the colon left out): typedUrlCredentials.
178
+ const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""];
179
+ const sources = [...values.flatMap(urlsWithin).map(credentialsIn), ...baseUrls.map(typedUrlCredentials)];
187
180
  const secrets = new Set();
188
181
  const encoded = new Set();
189
182
  const echoed = new Set();
190
183
  const passwords = new Set();
191
- for (const source of sources) {
192
- for (const secret of credentialsIn(source)) {
184
+ for (const found of sources) {
185
+ for (const secret of found) {
193
186
  secrets.add(secret);
194
187
  secrets.add(JSON.stringify(secret).slice(1, -1));
195
188
  // As commander's echo and the library's quoteValue show it: invisible characters escaped.
@@ -216,7 +209,7 @@ export function redactionFor(argv, env) {
216
209
  const echoedList = [...echoed].sort((a, b) => b.length - a.length);
217
210
  const passwordList = [...passwords].sort((a, b) => b.length - a.length);
218
211
  const out = (text) => {
219
- let result = redactUserinfo(redactCredentials(text, list));
212
+ let result = redactUrlUserinfo(redactCredentials(text, list));
220
213
  for (const form of encoded)
221
214
  result = result.split(form).join("***%40");
222
215
  return redactSecrets(result, echoedList);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.7.0",
3
+ "version": "0.8.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {