@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/README.md +21 -7
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
  15. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
  16. package/dist/src/cli/commands/catalogue.js +12 -11
  17. package/dist/src/cli/commands/portal.js +28 -20
  18. package/dist/src/cli/io.d.ts +16 -0
  19. package/dist/src/cli/io.js +55 -1
  20. package/dist/src/cli/output.d.ts +26 -0
  21. package/dist/src/cli/output.js +151 -0
  22. package/dist/src/cli/run.d.ts +0 -7
  23. package/dist/src/cli/run.js +80 -17
  24. package/dist/src/cli/shared.d.ts +14 -25
  25. package/dist/src/cli/shared.js +40 -102
  26. package/package.json +1 -1
@@ -1,12 +1,17 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
+ import { constants } from "node:buffer";
4
5
  import { TextDecoder } from "node:util";
5
6
  import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
6
7
  import { buildQueryString } from "./query.js";
7
- import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
8
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, cutText, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, TextBuilder, } from "./errors.js";
8
9
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
10
+ // quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
11
+ export { quoteValue } from "./errors.js";
9
12
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
13
+ /** V8's longest string (2^29 − 24 characters on 64-bit). */
14
+ const MAX_STRING_LENGTH = constants.MAX_STRING_LENGTH;
10
15
  const DEFAULT_USER_AGENT = "ckan-cli";
11
16
  const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
12
17
  /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
@@ -75,35 +80,57 @@ export function isBidiControl(code) {
75
80
  (code >= 0x202a && code <= 0x202e) ||
76
81
  (code >= 0x2066 && code <= 0x2069));
77
82
  }
83
+ /**
84
+ * The most server text (in UTF-16 units) `sanitizeServerText` reads; a longer text is cut first.
85
+ * Every message keeps far less of it (MAX_MESSAGE_VALUE_LENGTH, 500 characters), and cleaning all
86
+ * of a 16 MB error detail, one string node per character, exhausted a small heap.
87
+ */
88
+ const MAX_SERVER_TEXT = 16 * 1024;
89
+ /**
90
+ * A whole terminal escape sequence: a CSI (`ESC [` or the 8-bit U+009B, parameters,
91
+ * intermediates, final byte), a terminated control string (OSC/DCS/SOS/PM/APC, `ESC ]` …
92
+ * up to BEL or ST, or their 8-bit forms) or a short `ESC` sequence. Dropping only the
93
+ * control bytes left the printable rest behind (`ESC ] 0 ; title BEL` became `]0;title`).
94
+ * An unterminated control string is left to the per-character filter, so it cannot
95
+ * swallow the rest of the message. A control string's body ends at an ESC or at another
96
+ * introducer: with the introducers allowed inside it, an unterminated run of them was matched
97
+ * from each one to the end of the text, quadratic (100 000 × U+009D took 16 s). Written with
98
+ * escapes, so the source holds no control bytes.
99
+ */
100
+ const ESCAPE_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]|(?:\u001b[\]PX^_]|[\u0090\u0098\u009d-\u009f])[^\u0007\u001b\u0090\u0098\u009c-\u009f]*(?:\u0007|\u001b\\|\u009c)|\u001b[ -/]*[0-~]/g;
101
+ /**
102
+ * What `sanitizeServerText` drops once the escape sequences are gone: the C0 controls but the
103
+ * whitespace ones (TAB to CR), DEL, the C1 controls, and every format character (General
104
+ * Category Cf: bidi controls, zero-width characters, U+00AD, U+FEFF, …).
105
+ */
106
+ const DROPPED_CHAR = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f\p{Cf}]/gu;
78
107
  /**
79
108
  * Make a string that originates in an attacker-controlled response — the error
80
109
  * `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
81
110
  * into an error message on stderr:
82
111
  *
83
- * - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
112
+ * - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
113
+ * then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
84
114
  * (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
85
115
  * or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
86
116
  * spoofing, title changes, OSC 52 clipboard writes).
87
- * - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
88
- * reorder the visible message.
117
+ * - Unicode format characters (General Category Cf) are dropped: the bidi controls
118
+ * (isBidiControl), so server text cannot reorder the visible message, and the
119
+ * invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
120
+ * two messages that look identical are identical.
89
121
  * - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
90
122
  * space and the ends are trimmed, so the text stays on one line and a server
91
123
  * cannot forge an `Error:` line of its own.
124
+ * - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
125
+ * regex replace: a hostile error body costs neither seconds nor the heap. A caller that
126
+ * scrubs credentials does so first, so a cut never splits one.
92
127
  *
93
128
  * The CLI's JSON output is escaped separately (`escapeControlChars` in
94
129
  * cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
95
- * Written as a char-code filter so no raw control byte appears in this source.
130
+ * Written with escapes so no raw control byte appears in this source.
96
131
  */
97
132
  export function sanitizeServerText(text) {
98
- let out = "";
99
- for (const ch of text) {
100
- const n = ch.codePointAt(0) ?? 0;
101
- const whitespaceControl = n >= 0x09 && n <= 0x0d;
102
- if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
103
- continue;
104
- out += ch;
105
- }
106
- return out.replace(/\s+/g, " ").trim();
133
+ return cutText(text, MAX_SERVER_TEXT).replace(ESCAPE_SEQUENCE, "").replace(DROPPED_CHAR, "").replace(/\s+/g, " ").trim();
107
134
  }
108
135
  /**
109
136
  * Check a base URL (baseUrlProblem: not blank, no whitespace or control
@@ -200,14 +227,16 @@ function bodyBytes(value) {
200
227
  */
201
228
  function plainHeaders(headers) {
202
229
  const h = headers;
230
+ // Records built from an answer have no prototype: a header named `__proto__` is an own
231
+ // entry like any other, never the record's prototype (whose entries would read as headers).
203
232
  if (typeof h.get === "function" && typeof h.forEach === "function") {
204
- const record = {};
233
+ const record = Object.create(null);
205
234
  h.forEach.call(headers, (value, name) => {
206
235
  record[String(name).toLowerCase()] = value;
207
236
  });
208
237
  return record;
209
238
  }
210
- const record = {};
239
+ const record = Object.create(null);
211
240
  for (const [name, value] of Object.entries(headers)) {
212
241
  record[name.toLowerCase()] = value;
213
242
  }
@@ -218,11 +247,29 @@ function headerValue(value) {
218
247
  return Array.isArray(value) ? value[0] : value;
219
248
  }
220
249
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
250
+ /** No declared big-integer members (`EngineOptions.bigIntegerKeys` left out). */
251
+ const NO_KEYS = new Set();
252
+ /**
253
+ * Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
254
+ * copied; anything else throws an CkanValidationError.
255
+ */
256
+ export function bigIntegerKeysOption(value) {
257
+ if (value === undefined)
258
+ return NO_KEYS;
259
+ const iterable = typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
260
+ const names = iterable ? [...value] : [];
261
+ if (!iterable || names.some((key) => typeof key !== "string" || key === "")) {
262
+ throw new CkanValidationError("Invalid option bigIntegerKeys: expected a set of non-empty member names.");
263
+ }
264
+ return new Set(names);
265
+ }
221
266
  export class RequestEngine {
222
267
  // A real private field (not TypeScript's `private`): util.inspect, console.log and
223
268
  // JSON.stringify of a client never show it, so a password in the base URL can't be
224
269
  // logged by accident. Messages use redactUrl.
225
270
  #baseUrl;
271
+ /** `#baseUrl` without its userinfo: what every request URL starts with. */
272
+ #requestBase;
226
273
  /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
227
274
  #credentials;
228
275
  /**
@@ -240,14 +287,27 @@ export class RequestEngine {
240
287
  maxResponseBytes;
241
288
  sleep;
242
289
  onRetry;
290
+ onWarning;
291
+ /** `EngineOptions.bigIntegerKeys`, copied. */
292
+ bigIntegerKeys;
243
293
  constructor(options = {}) {
244
294
  // The raw value, checked before the slash strip (the engine glues it into
245
295
  // every URL), and here rather than only in the default transport: a library
246
296
  // consumer that injects a custom transport would otherwise get no gating at
247
297
  // all, and could be steered to a non-http(s) scheme. Only undefined selects
248
298
  // the default.
249
- this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
250
- this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
299
+ const baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
300
+ // Requests go to the base URL as WHATWG parses it, the reading every check of it makes (the
301
+ // host, the cleartext warning, the Authorization header), never to a text-based reading of
302
+ // the string: a transport that reads the URL as text reaches the same host.
303
+ const parsed = new URL(baseUrl);
304
+ this.#baseUrl = parsed.href.replace(/\/+$/, "");
305
+ parsed.username = "";
306
+ parsed.password = "";
307
+ this.#requestBase = parsed.href.replace(/\/+$/, "");
308
+ // The userinfo as typed and as sent (WHATWG percent-encodes some of its characters).
309
+ const userinfos = [...new Set([...credentialsIn(baseUrl), ...credentialsIn(this.#baseUrl)])];
310
+ this.#credentials = userinfos.flatMap((raw) => {
251
311
  try {
252
312
  return [raw, decodeURIComponent(raw)];
253
313
  }
@@ -255,8 +315,7 @@ export class RequestEngine {
255
315
  return [raw];
256
316
  }
257
317
  });
258
- this.#echoed = credentialsIn(this.#baseUrl)
259
- .flatMap(echoedCredentialForms)
318
+ this.#echoed = [...new Set(userinfos.flatMap(echoedCredentialForms).flatMap(urlSpellings))]
260
319
  .sort((a, b) => b.length - a.length);
261
320
  this.transport = functionOption("transport", options.transport, nodeHttpTransport);
262
321
  // Only undefined selects the default; a blank or unsendable value is refused.
@@ -273,6 +332,18 @@ export class RequestEngine {
273
332
  this.sleep = functionOption("sleep", options.sleep, realSleep);
274
333
  this.onRetry =
275
334
  options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
335
+ this.onWarning =
336
+ options.onWarning === undefined ? undefined : functionOption("onWarning", options.onWarning, () => { });
337
+ this.bigIntegerKeys = bigIntegerKeysOption(options.bigIntegerKeys);
338
+ }
339
+ /** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
340
+ warn(message) {
341
+ try {
342
+ this.onWarning?.(message);
343
+ }
344
+ catch {
345
+ // a logging hook is no reason to fail the request
346
+ }
276
347
  }
277
348
  /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
278
349
  async backOff(attempt, delayMs, url, status) {
@@ -300,6 +371,34 @@ export class RequestEngine {
300
371
  scrub(text) {
301
372
  return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
302
373
  }
374
+ /**
375
+ * An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
376
+ * with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
377
+ * credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
378
+ * a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
379
+ * with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
380
+ * decodes, and stays as sent; one too deeply nested to write again is left out.
381
+ */
382
+ scrubBody(scrubbed) {
383
+ if (this.#echoed.length === 0 || !scrubbed.includes("\\"))
384
+ return scrubbed;
385
+ let decoded;
386
+ try {
387
+ decoded = JSON.parse(scrubbed);
388
+ }
389
+ catch {
390
+ return scrubbed;
391
+ }
392
+ let rewritten;
393
+ try {
394
+ rewritten = JSON.stringify(decoded);
395
+ }
396
+ catch {
397
+ return "";
398
+ }
399
+ const spelled = this.#echoed.map((form) => JSON.stringify(form).slice(1, -1));
400
+ return spelled.some((form) => rewritten.includes(form)) ? redactSecrets(rewritten, spelled) : scrubbed;
401
+ }
303
402
  /**
304
403
  * A failure as the `cause` of the error the engine raises: the original when its
305
404
  * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
@@ -337,7 +436,7 @@ export class RequestEngine {
337
436
  composeUrl(path, query, withUserinfo) {
338
437
  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
339
438
  const qs = query ? buildQueryString(query) : "";
340
- const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
439
+ const base = withUserinfo ? this.#baseUrl : this.#requestBase;
341
440
  return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
342
441
  }
343
442
  /**
@@ -367,6 +466,24 @@ export class RequestEngine {
367
466
  clearTimeout(timer);
368
467
  }
369
468
  }
469
+ /**
470
+ * The URL a redirect leads to as messages and errors show it: the Location with the base URL's
471
+ * credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
472
+ * spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
473
+ * left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
474
+ */
475
+ shownTarget(location, shown) {
476
+ const sent = this.scrub(location);
477
+ try {
478
+ const target = new URL(sent, shown);
479
+ target.username = "";
480
+ target.password = "";
481
+ return target.href;
482
+ }
483
+ catch {
484
+ return sent;
485
+ }
486
+ }
370
487
  /** Perform a request with Accept negotiation and transient-error retries. */
371
488
  async request(method, path, options = { accept: "application/json" }) {
372
489
  // The transport never sees the base URL's userinfo: the engine sends it as an
@@ -374,6 +491,9 @@ export class RequestEngine {
374
491
  // absolute) keeps it and one to another origin or scheme drops it. A transport
375
492
  // such as fetch also refuses a URL with credentials outright.
376
493
  let url = this.composeUrl(path, options.query, false);
494
+ // The URL as messages, errors and `RawResponse.url` show it: after a redirect, the Location
495
+ // with the base URL's credentials scrubbed (shownTarget); `url` is what is requested.
496
+ let shown = url;
377
497
  let headers = {
378
498
  Accept: options.accept,
379
499
  "User-Agent": this.userAgent,
@@ -407,7 +527,7 @@ export class RequestEngine {
407
527
  if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
408
528
  throw cause;
409
529
  const reason = cause instanceof Error ? cause.message : String(cause);
410
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
530
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
411
531
  cause: this.scrubCause(cause),
412
532
  });
413
533
  }
@@ -415,14 +535,14 @@ export class RequestEngine {
415
535
  // would otherwise surface below as a raw TypeError, outside the CkanError contract.
416
536
  const invalid = responseProblem(response);
417
537
  if (invalid !== undefined) {
418
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
538
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport returned an invalid response (${invalid}).`);
419
539
  }
420
540
  // A transport must not follow redirects itself (`redirect: "manual"`): one that
421
541
  // did (fetch's default) may have carried the Authorization header to another
422
542
  // host, and the answer is not the one asked for. Reject it when it says so (`url`).
423
543
  const finalUrl = response.url;
424
544
  if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
425
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
545
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport followed a redirect to another origin ` +
426
546
  `(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
427
547
  `(HttpRequest.redirect is "manual").`);
428
548
  }
@@ -433,7 +553,7 @@ export class RequestEngine {
433
553
  // The size cap holds whatever the transport did: the default one aborts early, a
434
554
  // custom one may have read everything.
435
555
  if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
436
- throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
556
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
437
557
  }
438
558
  const retryable = status === 429 || status === 503;
439
559
  // A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
@@ -446,7 +566,7 @@ export class RequestEngine {
446
566
  // for less: `Retry-After: 0` or a date in the past turned the retries into a
447
567
  // zero-delay burst against a server that had just asked for less load.
448
568
  const backoff = this.retryDelayMs * attempt;
449
- await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
569
+ await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), shown, status);
450
570
  continue;
451
571
  }
452
572
  // Follow redirects, resolving the Location relative to the current URL.
@@ -462,7 +582,7 @@ export class RequestEngine {
462
582
  // never reaches a transport (a custom one may not check the scheme).
463
583
  if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
464
584
  throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
465
- `for ${method} ${cutForMessage(redactUrl(url))}`);
585
+ `for ${method} ${cutForMessage(redactUrl(shown))}`);
466
586
  }
467
587
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
468
588
  // port, so an https->http downgrade counts), keep only the engine's own
@@ -483,6 +603,7 @@ export class RequestEngine {
483
603
  headers = engineHeadersOnly(headers);
484
604
  }
485
605
  url = nextUrl.toString();
606
+ shown = this.shownTarget(location, shown);
486
607
  redirects += 1;
487
608
  continue;
488
609
  }
@@ -497,47 +618,72 @@ export class RequestEngine {
497
618
  const detail = target !== undefined
498
619
  ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
499
620
  : location
500
- ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
621
+ ? `redirect to ${cutForMessage(sanitizeServerText(this.scrub(location)))} not followed`
501
622
  : "redirect not followed (no Location header)";
502
- throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
623
+ throw new CkanApiError({ status, url: shown, method, body: this.scrubBody(this.scrub(errorBodyText(body))), detail });
503
624
  }
504
- throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
625
+ throw this.toApiError(method, shown, status, body, status === 401 || status === 403 ? dropped : undefined, {
505
626
  retries: attempt,
506
627
  ...(tooLong ? { retryAfterMs: retryAfter } : {}),
507
628
  });
508
629
  }
509
- return { data: body, contentType, status, url };
630
+ return { data: body, contentType, status, url: shown };
510
631
  }
511
632
  }
512
- /** Perform a GET expecting JSON and parse it into `T`. */
633
+ /** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
513
634
  async getJson(path, query) {
635
+ return (await this.getAnswer(path, query)).value;
636
+ }
637
+ /**
638
+ * Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
639
+ * userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
640
+ * MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
641
+ * (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
642
+ * one failed; the client prefixes its own envelope and shape errors with it too.
643
+ *
644
+ * The text is never changed silently: bytes that are not valid in the charset, and lone
645
+ * surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
646
+ * well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
647
+ * `onWarning` is told how many. One bad character never sinks a whole answer.
648
+ */
649
+ async getAnswer(path, query) {
514
650
  const res = await this.request("GET", path, { query, accept: "application/json" });
515
- const text = decodeBody(res.data, res.contentType, res.url);
651
+ const url = redactUrl(res.url);
652
+ const request = `GET ${cutForMessage(url)}`;
653
+ const decoded = decodeBody(res.data, res.contentType, request);
654
+ if (decoded.text.trim() === "") {
655
+ // A broken answer, not "no results".
656
+ throw new CkanParseError(`${request}: empty response body.`);
657
+ }
658
+ const repaired = wellFormedEscapes(decoded.text);
659
+ // Declared big integers are quoted first, so JSON.parse delivers them exactly.
660
+ const { json, inexact } = scanNumbers(repaired.json, this.bigIntegerKeys);
516
661
  let parsed;
517
662
  try {
518
- parsed = JSON.parse(text);
663
+ parsed = JSON.parse(json);
519
664
  }
520
665
  catch (cause) {
521
- // Name the URL and what came back: a site that is not a CKAN (or not at this
522
- // path) typically answers with an HTML page and HTTP 200.
523
- // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
524
- // 8-bit CSI U+009B): sanitise it before it reaches stderr.
525
- const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
526
- const message = /json/i.test(mediaType)
527
- ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
528
- : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
529
- throw new CkanParseError(message, { cause: this.scrubCause(cause) });
666
+ // A site that is not a CKAN (or not at this path) typically answers with an HTML page and
667
+ // HTTP 200: name its type. The Content-Type is server text (latin-1 decoded, so a 0x9B byte
668
+ // is the 8-bit CSI U+009B): sanitise it before it reaches stderr.
669
+ const type = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
670
+ const hint = type !== "" && !/json/i.test(type) ? ` (expected JSON, got Content-Type "${type}")` : "";
671
+ throw new CkanParseError(`${request}: the answer is not valid JSON${hint}.`, { cause: this.scrubCause(cause) });
530
672
  }
531
673
  // JSON.parse rounds and overflows numbers silently: refuse the answer rather than print a
532
674
  // changed value.
533
- const inexact = firstInexactNumber(text);
534
675
  if (inexact !== undefined) {
535
- throw new CkanParseError(`Unexpected response from ${cutForMessage(redactUrl(res.url))}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
676
+ throw new CkanParseError(`${request}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
536
677
  }
537
- return parsed;
678
+ const replaced = decoded.replaced + repaired.replaced;
679
+ if (replaced > 0) {
680
+ this.warn(`${request}: ${replaced} ${replaced === 1 ? "character" : "characters"} of the answer ${replaced === 1 ? "was" : "were"} ` +
681
+ "not valid text (invalid bytes or a lone surrogate) and replaced with U+FFFD.");
682
+ }
683
+ return { value: parsed, url, request };
538
684
  }
539
685
  toApiError(method, url, status, body, hint, retry) {
540
- const text = this.scrub(body.toString("utf8"));
686
+ const text = this.scrub(errorBodyText(body));
541
687
  let detail;
542
688
  try {
543
689
  const parsed = JSON.parse(text);
@@ -564,28 +710,24 @@ export class RequestEngine {
564
710
  catch {
565
711
  // Non-JSON error body; leave detail undefined.
566
712
  }
567
- // `detail` came from the response body; strip control characters so a hostile
713
+ // `detail` came from the response body, decoded: scrubbed again (a JSON escape hid the
714
+ // credential from the scrub above), then stripped of control characters so a hostile
568
715
  // endpoint cannot inject terminal escape sequences via the stderr error message.
569
716
  if (detail !== undefined)
570
- detail = sanitizeServerText(detail);
717
+ detail = sanitizeServerText(this.scrub(detail));
571
718
  if (hint !== undefined)
572
719
  detail = detail === undefined ? hint : `${detail}; ${hint}`;
573
720
  return new CkanApiError({
574
721
  status,
575
722
  url,
576
723
  method,
577
- body: text,
724
+ body: this.scrubBody(text),
578
725
  detail,
579
726
  retries: retry.retries,
580
727
  ...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
581
728
  });
582
729
  }
583
730
  }
584
- /** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
585
- function withoutUserinfo(url) {
586
- const [userinfo] = credentialsIn(url);
587
- return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
588
- }
589
731
  /**
590
732
  * The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
591
733
  * both percent-decoded, as Node's own http client builds it), or undefined without
@@ -611,18 +753,143 @@ function originOf(url) {
611
753
  * Decode a response body by the charset of its Content-Type (UTF-8 when none is
612
754
  * given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
613
755
  * that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
614
- * sends UTF-8; this matters for proxies and mirrors that re-encode.
756
+ * sends UTF-8; this matters for proxies and mirrors that re-encode. An unknown charset
757
+ * label is a CkanParseError naming `request`. Bytes that are not valid in the charset become
758
+ * U+FFFD, and `replaced` counts them: exactly for UTF-8 and the single-byte charsets; for
759
+ * UTF-16 a U+FFFD the server sent counts too.
615
760
  */
616
- function decodeBody(body, contentType, url) {
761
+ function decodeBody(body, contentType, request) {
617
762
  const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
618
763
  let decoder;
619
764
  try {
620
- decoder = new TextDecoder(charset);
765
+ decoder = new TextDecoder(charset, { fatal: true });
766
+ }
767
+ catch {
768
+ throw new CkanParseError(`${request}: unsupported response charset "${cutForMessage(sanitizeServerText(charset))}".`);
769
+ }
770
+ try {
771
+ return { text: decoder.decode(body), replaced: 0 };
772
+ }
773
+ catch (err) {
774
+ if (isStringTooLong(err))
775
+ throw tooLargeForText(request, body);
776
+ // Not valid in the charset: decode with replacement, and count what was replaced.
777
+ let text;
778
+ try {
779
+ text = new TextDecoder(decoder.encoding).decode(body);
780
+ }
781
+ catch (again) {
782
+ if (isStringTooLong(again))
783
+ throw tooLargeForText(request, body);
784
+ throw again;
785
+ }
786
+ const sent = decoder.encoding === "utf-8" ? countBytes(body, REPLACEMENT_UTF8) : 0;
787
+ return { text, replaced: countChar(text, 0xfffd) - sent };
788
+ }
789
+ }
790
+ /**
791
+ * True for V8's error for a string past its longest one (2^29 − 24 characters on 64-bit): a body
792
+ * of 512 MB and more, possible once `maxResponseBytes` is lifted, can't be decoded into one.
793
+ */
794
+ function isStringTooLong(err) {
795
+ return err?.code === "ERR_STRING_TOO_LONG";
796
+ }
797
+ /** The parse error for a body too large to decode: it used to escape as a plain Error ("Unexpected error"). */
798
+ function tooLargeForText(request, body) {
799
+ return new CkanParseError(`${request}: the answer is too large to read as text (${body.byteLength} bytes; the longest string is ${MAX_STRING_LENGTH} characters).`);
800
+ }
801
+ /**
802
+ * The body of an error answer as text, kept on the error (`body`). One too large to be a string
803
+ * (maxResponseBytes lifted) is left out: the status is the error.
804
+ */
805
+ function errorBodyText(body) {
806
+ try {
807
+ return body.toString("utf8");
808
+ }
809
+ catch (err) {
810
+ if (!isStringTooLong(err))
811
+ throw err;
812
+ return "";
813
+ }
814
+ }
815
+ /** U+FFFD in UTF-8. */
816
+ const REPLACEMENT_UTF8 = Buffer.from([0xef, 0xbf, 0xbd]);
817
+ /** How often `needle` occurs in `body`. */
818
+ function countBytes(body, needle) {
819
+ let n = 0;
820
+ for (let at = body.indexOf(needle); at !== -1; at = body.indexOf(needle, at + needle.length))
821
+ n++;
822
+ return n;
823
+ }
824
+ /** How often the UTF-16 unit `code` occurs in `text`. */
825
+ function countChar(text, code) {
826
+ let n = 0;
827
+ for (let i = 0; i < text.length; i++)
828
+ if (text.charCodeAt(i) === code)
829
+ n++;
830
+ return n;
831
+ }
832
+ /** A JSON escape of a surrogate, `\uD800`–`\uDFFF`, anywhere in a text (a cheap pre-check). */
833
+ const SURROGATE_ESCAPE = /\\u[dD][89a-fA-F][0-9a-fA-F]{2}/;
834
+ /** The value of the four hex digits of a `\uXXXX` escape whose `u` is at `at`, or -1. */
835
+ function escapeAt(json, at) {
836
+ if (json.charCodeAt(at) !== 0x75)
837
+ return -1;
838
+ const hex = json.slice(at + 1, at + 5);
839
+ return /^[0-9a-fA-F]{4}$/.test(hex) ? parseInt(hex, 16) : -1;
840
+ }
841
+ /**
842
+ * `json` (decoded answer text) with every lone surrogate written as a JSON escape (`\ud800` not
843
+ * followed by a low-surrogate escape, or a `\udc00` not preceded by a high one) rewritten as
844
+ * `\ufffd`, and how many. JSON.parse would make such an escape half a character, which a JSON
845
+ * printer writes back as `\ud800` and `jq` refuses (the whole output with it). A raw lone
846
+ * surrogate cannot occur: TextDecoder never makes one. One linear pass, and only when the text
847
+ * holds a surrogate escape at all; the text is copied only when one is lone.
848
+ */
849
+ function wellFormedEscapes(json) {
850
+ if (!SURROGATE_ESCAPE.test(json))
851
+ return { json, replaced: 0 };
852
+ const out = new TextBuilder();
853
+ let copied = 0;
854
+ let replaced = 0;
855
+ for (let i = 0; i < json.length; i++) {
856
+ if (json.charCodeAt(i) !== 0x5c)
857
+ continue;
858
+ const code = escapeAt(json, i + 1);
859
+ if (code < 0xd800 || code > 0xdfff) {
860
+ i++; // an escape of another kind (\", \\, \n, \u0041): skip its second character
861
+ continue;
862
+ }
863
+ const pairedLow = code <= 0xdbff && json.charCodeAt(i + 6) === 0x5c && isLowSurrogate(escapeAt(json, i + 7));
864
+ if (pairedLow) {
865
+ i += 11;
866
+ continue;
867
+ }
868
+ out.push(json.slice(copied, i), "\\ufffd");
869
+ copied = i + 6;
870
+ replaced++;
871
+ i += 5;
872
+ }
873
+ if (replaced === 0)
874
+ return { json, replaced };
875
+ out.push(json.slice(copied));
876
+ return { json: out.toString(), replaced };
877
+ }
878
+ function isLowSurrogate(code) {
879
+ return code >= 0xdc00 && code <= 0xdfff;
880
+ }
881
+ /**
882
+ * `form` and its spelling in a URL (`encodeURIComponent`): a server that puts a credential into
883
+ * a URL, a redirect's Location, percent-encodes it. A form that has no such spelling (a lone
884
+ * surrogate) is kept as it is.
885
+ */
886
+ function urlSpellings(form) {
887
+ try {
888
+ return [form, encodeURIComponent(form)];
621
889
  }
622
890
  catch {
623
- throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
891
+ return [form];
624
892
  }
625
- return decoder.decode(body);
626
893
  }
627
894
  /** Resolve a Location header against the current URL; undefined if missing or malformed. */
628
895
  function resolveLocation(location, base) {
@@ -637,16 +904,20 @@ function resolveLocation(location, base) {
637
904
  }
638
905
  /** A JSON number token, matched at a given position (sticky). */
639
906
  const JSON_NUMBER = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
907
+ /** The smallest positive normal double, 2^-1022. Below it a double has fewer significant digits. */
908
+ const MIN_NORMAL = 2.2250738585072014e-308;
640
909
  /**
641
910
  * Whether the double `value` (finite) prints as the number `literal` wrote, checked where a
642
- * double can be off by a whole number or more: past 2^53, where every double is an integer, and
643
- * at 0, which a literal too small for a double becomes. Past 2^53 the CLI prints `String(value)`,
644
- * the shortest decimal that reads back as the same double: `12345678901234567890` prints as
645
- * `12345678901234567000`, a changed value, while `1E23` or `1.7976931348623157E308` (a double a
646
- * server printed in its own shortest form) print as the same number.
911
+ * double can be off in its leading digits: past 2^53, where every double is an integer, at 0,
912
+ * which a literal too small for a double becomes, and below the smallest normal double, where a
913
+ * subnormal has fewer significant digits (`3e-324` becomes `5e-324`). There the CLI prints
914
+ * `String(value)`, the shortest decimal that reads back as the same double: `12345678901234567890`
915
+ * prints as `12345678901234567000`, a changed value, while `1E23`, `1.7976931348623157E308` or
916
+ * `5e-324` (a double a server printed in its own shortest form) print as the same number.
647
917
  */
648
918
  function exactAtTheEdges(literal, value) {
649
- if (value !== 0 && Math.abs(value) <= Number.MAX_SAFE_INTEGER)
919
+ const size = Math.abs(value);
920
+ if (value !== 0 && size <= Number.MAX_SAFE_INTEGER && size >= MIN_NORMAL)
650
921
  return true;
651
922
  const written = decimalOf(literal);
652
923
  if (value === 0)
@@ -657,30 +928,87 @@ function exactAtTheEdges(literal, value) {
657
928
  /** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
658
929
  function decimalOf(text) {
659
930
  const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
660
- const all = (whole + fraction).replace(/^0+/, "");
661
- const digits = all.replace(/0+$/, "");
662
- return { negative: sign === "-", digits, exponent: Number(exp) - fraction.length + (all.length - digits.length) };
931
+ // The zeros are counted with index loops: `/0+$/` tried a match at every zero of a run that
932
+ // doesn't end the text, quadratic in the run, and one `1.<64 000 zeros>1e300` in an answer held
933
+ // the CLI for seconds, past --timeout (the scan runs once the answer is in).
934
+ const all = whole + fraction;
935
+ let start = 0;
936
+ while (start < all.length && all.charCodeAt(start) === 0x30)
937
+ start++;
938
+ let end = all.length;
939
+ while (end > start && all.charCodeAt(end - 1) === 0x30)
940
+ end--;
941
+ return { negative: sign === "-", digits: all.slice(start, end), exponent: Number(exp) - fraction.length + (all.length - end) };
942
+ }
943
+ /** A JSON number token as the grammar allows it, matched at a given position (sticky). */
944
+ const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
945
+ /** Whether the member name `json` holds between `start` and `end` (its quotes) is one of `keys`, compared in place. */
946
+ function isDeclared(json, start, end, keys) {
947
+ for (const key of keys) {
948
+ if (end - start - 1 === key.length && json.startsWith(key, start + 1))
949
+ return true;
950
+ }
951
+ return false;
952
+ }
953
+ /** The index of the first character at or after `i` that is not JSON whitespace. */
954
+ function skipSpace(json, i) {
955
+ for (; i < json.length; i++) {
956
+ const c = json.charCodeAt(i);
957
+ if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d)
958
+ break;
959
+ }
960
+ return i;
663
961
  }
664
962
  /**
665
- * The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
666
- * undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
667
- * 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
668
- * `9007199254740993e0`, `9007199254740992.5`), and a non-zero one too small for a double
669
- * (`1e-400` became 0). Below 2^53 a decimal fraction keeps a double's usual precision, as JSON
670
- * means it. A linear scan that skips strings; a reviver would recurse per nesting level.
963
+ * One linear pass over `json` (the text of an answer) that skips strings and looks at every
964
+ * number literal:
965
+ * - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
966
+ * 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
967
+ * it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
968
+ * so a malformed one (`01`) still fails JSON.parse;
969
+ * - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
970
+ * cannot carry exactly is reported in `inexact`.
971
+ * Nothing is allocated per value: a member name is compared in place (only a name written with
972
+ * an escape is decoded), and the text is copied only when a declared member is found. No
973
+ * reviver, which would recurse per nesting level and overflow on deep data.
671
974
  */
672
- function firstInexactNumber(json) {
975
+ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
976
+ const keys = bigIntegerKeys.size === 0 ? undefined : [...bigIntegerKeys];
977
+ let inexact;
978
+ let out;
979
+ let copied = 0;
673
980
  for (let i = 0; i < json.length; i++) {
674
981
  const c = json.charCodeAt(i);
675
982
  if (c === 0x22) {
676
983
  // a string: skip to its closing quote, past escapes
984
+ const start = i;
985
+ let escaped = false;
677
986
  for (i++; i < json.length; i++) {
678
987
  const d = json.charCodeAt(i);
679
- if (d === 0x5c)
988
+ if (d === 0x5c) {
989
+ escaped = true;
680
990
  i++;
991
+ }
681
992
  else if (d === 0x22)
682
993
  break;
683
994
  }
995
+ if (keys === undefined)
996
+ continue;
997
+ // A member name is a string followed by a colon; is it a declared one with a number value?
998
+ const colon = skipSpace(json, i + 1);
999
+ if (json.charCodeAt(colon) !== 0x3a)
1000
+ continue;
1001
+ if (!(escaped ? bigIntegerKeys.has(decodedName(json.slice(start, i + 1))) : isDeclared(json, start, i, keys)))
1002
+ continue;
1003
+ const at = skipSpace(json, colon + 1);
1004
+ STRICT_JSON_NUMBER.lastIndex = at;
1005
+ const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
1006
+ if (literal === undefined)
1007
+ continue;
1008
+ out ??= new TextBuilder();
1009
+ out.push(json.slice(copied, at), '"', literal, '"');
1010
+ copied = at + literal.length;
1011
+ i = copied - 1;
684
1012
  }
685
1013
  else if (c === 0x2d || (c >= 0x30 && c <= 0x39)) {
686
1014
  JSON_NUMBER.lastIndex = i;
@@ -688,10 +1016,37 @@ function firstInexactNumber(json) {
688
1016
  if (literal === undefined)
689
1017
  continue;
690
1018
  const value = Number(literal);
691
- if (!Number.isFinite(value) || !exactAtTheEdges(literal, value))
692
- return literal;
1019
+ if (inexact === undefined && (!Number.isFinite(value) || !exactAtTheEdges(literal, value))) {
1020
+ inexact = literal;
1021
+ if (keys === undefined)
1022
+ break;
1023
+ }
693
1024
  i += literal.length - 1;
694
1025
  }
695
1026
  }
696
- return undefined;
1027
+ if (out === undefined)
1028
+ return { json, inexact };
1029
+ out.push(json.slice(copied));
1030
+ return { json: out.toString(), inexact };
1031
+ }
1032
+ /** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
1033
+ function decodedName(quoted) {
1034
+ try {
1035
+ return String(JSON.parse(quoted));
1036
+ }
1037
+ catch {
1038
+ return "";
1039
+ }
1040
+ }
1041
+ /**
1042
+ * The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
1043
+ * undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
1044
+ * 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
1045
+ * `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
1046
+ * (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
1047
+ * `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
1048
+ * precision, as JSON means it. {@link scanNumbers} without declared members.
1049
+ */
1050
+ export function firstInexactNumber(json) {
1051
+ return scanNumbers(json).inexact;
697
1052
  }