@maschinenlesbar.org/ckan-cli 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. package/README.md +25 -9
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +17 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +102 -49
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +90 -11
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +337 -21
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +16 -1
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +33 -1
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +10 -7
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/types.d.ts +11 -11
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +73 -7
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +180 -16
  15. package/dist/src/cli/commands/catalogue.js +12 -11
  16. package/dist/src/cli/commands/portal.js +28 -20
  17. package/dist/src/cli/io.d.ts +16 -0
  18. package/dist/src/cli/io.js +55 -1
  19. package/dist/src/cli/output.d.ts +26 -0
  20. package/dist/src/cli/output.js +151 -0
  21. package/dist/src/cli/run.js +76 -6
  22. package/dist/src/cli/shared.d.ts +14 -25
  23. package/dist/src/cli/shared.js +40 -72
  24. package/package.json +1 -1
@@ -1,12 +1,17 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
+ import { constants } from "node:buffer";
4
5
  import { TextDecoder } from "node:util";
5
6
  import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
6
7
  import { buildQueryString } from "./query.js";
7
8
  import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
8
9
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
10
+ // quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
11
+ export { quoteValue } from "./errors.js";
9
12
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
13
+ /** V8's longest string (2^29 − 24 characters on 64-bit). */
14
+ const MAX_STRING_LENGTH = constants.MAX_STRING_LENGTH;
10
15
  const DEFAULT_USER_AGENT = "ckan-cli";
11
16
  const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
12
17
  /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
@@ -200,14 +205,16 @@ function bodyBytes(value) {
200
205
  */
201
206
  function plainHeaders(headers) {
202
207
  const h = headers;
208
+ // Records built from an answer have no prototype: a header named `__proto__` is an own
209
+ // entry like any other, never the record's prototype (whose entries would read as headers).
203
210
  if (typeof h.get === "function" && typeof h.forEach === "function") {
204
- const record = {};
211
+ const record = Object.create(null);
205
212
  h.forEach.call(headers, (value, name) => {
206
213
  record[String(name).toLowerCase()] = value;
207
214
  });
208
215
  return record;
209
216
  }
210
- const record = {};
217
+ const record = Object.create(null);
211
218
  for (const [name, value] of Object.entries(headers)) {
212
219
  record[name.toLowerCase()] = value;
213
220
  }
@@ -218,6 +225,22 @@ function headerValue(value) {
218
225
  return Array.isArray(value) ? value[0] : value;
219
226
  }
220
227
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
228
+ /** No declared big-integer members (`EngineOptions.bigIntegerKeys` left out). */
229
+ const NO_KEYS = new Set();
230
+ /**
231
+ * Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
232
+ * copied; anything else throws an CkanValidationError.
233
+ */
234
+ export function bigIntegerKeysOption(value) {
235
+ if (value === undefined)
236
+ return NO_KEYS;
237
+ const iterable = typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
238
+ const names = iterable ? [...value] : [];
239
+ if (!iterable || names.some((key) => typeof key !== "string" || key === "")) {
240
+ throw new CkanValidationError("Invalid option bigIntegerKeys: expected a set of non-empty member names.");
241
+ }
242
+ return new Set(names);
243
+ }
221
244
  export class RequestEngine {
222
245
  // A real private field (not TypeScript's `private`): util.inspect, console.log and
223
246
  // JSON.stringify of a client never show it, so a password in the base URL can't be
@@ -240,6 +263,9 @@ export class RequestEngine {
240
263
  maxResponseBytes;
241
264
  sleep;
242
265
  onRetry;
266
+ onWarning;
267
+ /** `EngineOptions.bigIntegerKeys`, copied. */
268
+ bigIntegerKeys;
243
269
  constructor(options = {}) {
244
270
  // The raw value, checked before the slash strip (the engine glues it into
245
271
  // every URL), and here rather than only in the default transport: a library
@@ -273,6 +299,18 @@ export class RequestEngine {
273
299
  this.sleep = functionOption("sleep", options.sleep, realSleep);
274
300
  this.onRetry =
275
301
  options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
302
+ this.onWarning =
303
+ options.onWarning === undefined ? undefined : functionOption("onWarning", options.onWarning, () => { });
304
+ this.bigIntegerKeys = bigIntegerKeysOption(options.bigIntegerKeys);
305
+ }
306
+ /** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
307
+ warn(message) {
308
+ try {
309
+ this.onWarning?.(message);
310
+ }
311
+ catch {
312
+ // a logging hook is no reason to fail the request
313
+ }
276
314
  }
277
315
  /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
278
316
  async backOff(attempt, delayMs, url, status) {
@@ -499,7 +537,7 @@ export class RequestEngine {
499
537
  : location
500
538
  ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
501
539
  : "redirect not followed (no Location header)";
502
- throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
540
+ throw new CkanApiError({ status, url, method, body: this.scrub(errorBodyText(body)), detail });
503
541
  }
504
542
  throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
505
543
  retries: attempt,
@@ -509,27 +547,60 @@ export class RequestEngine {
509
547
  return { data: body, contentType, status, url };
510
548
  }
511
549
  }
512
- /** Perform a GET expecting JSON and parse it into `T`. */
550
+ /** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
513
551
  async getJson(path, query) {
552
+ return (await this.getAnswer(path, query)).value;
553
+ }
554
+ /**
555
+ * Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
556
+ * userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
557
+ * MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
558
+ * (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
559
+ * one failed; the client prefixes its own envelope and shape errors with it too.
560
+ *
561
+ * The text is never changed silently: bytes that are not valid in the charset, and lone
562
+ * surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
563
+ * well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
564
+ * `onWarning` is told how many. One bad character never sinks a whole answer.
565
+ */
566
+ async getAnswer(path, query) {
514
567
  const res = await this.request("GET", path, { query, accept: "application/json" });
515
- const text = decodeBody(res.data, res.contentType, res.url);
568
+ const url = redactUrl(res.url);
569
+ const request = `GET ${cutForMessage(url)}`;
570
+ const decoded = decodeBody(res.data, res.contentType, request);
571
+ if (decoded.text.trim() === "") {
572
+ // A broken answer, not "no results".
573
+ throw new CkanParseError(`${request}: empty response body.`);
574
+ }
575
+ const repaired = wellFormedEscapes(decoded.text);
576
+ // Declared big integers are quoted first, so JSON.parse delivers them exactly.
577
+ const { json, inexact } = scanNumbers(repaired.json, this.bigIntegerKeys);
578
+ let parsed;
516
579
  try {
517
- return JSON.parse(text);
580
+ parsed = JSON.parse(json);
518
581
  }
519
582
  catch (cause) {
520
- // Name the URL and what came back: a site that is not a CKAN (or not at this
521
- // path) typically answers with an HTML page and HTTP 200.
522
- // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
523
- // 8-bit CSI U+009B): sanitise it before it reaches stderr.
524
- const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
525
- const message = /json/i.test(mediaType)
526
- ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
527
- : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
528
- throw new CkanParseError(message, { cause: this.scrubCause(cause) });
583
+ // A site that is not a CKAN (or not at this path) typically answers with an HTML page and
584
+ // HTTP 200: name its type. The Content-Type is server text (latin-1 decoded, so a 0x9B byte
585
+ // is the 8-bit CSI U+009B): sanitise it before it reaches stderr.
586
+ const type = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
587
+ const hint = type !== "" && !/json/i.test(type) ? ` (expected JSON, got Content-Type "${type}")` : "";
588
+ throw new CkanParseError(`${request}: the answer is not valid JSON${hint}.`, { cause: this.scrubCause(cause) });
529
589
  }
590
+ // JSON.parse rounds and overflows numbers silently: refuse the answer rather than print a
591
+ // changed value.
592
+ if (inexact !== undefined) {
593
+ throw new CkanParseError(`${request}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
594
+ }
595
+ const replaced = decoded.replaced + repaired.replaced;
596
+ if (replaced > 0) {
597
+ this.warn(`${request}: ${replaced} ${replaced === 1 ? "character" : "characters"} of the answer ${replaced === 1 ? "was" : "were"} ` +
598
+ "not valid text (invalid bytes or a lone surrogate) and replaced with U+FFFD.");
599
+ }
600
+ return { value: parsed, url, request };
530
601
  }
531
602
  toApiError(method, url, status, body, hint, retry) {
532
- const text = this.scrub(body.toString("utf8"));
603
+ const text = this.scrub(errorBodyText(body));
533
604
  let detail;
534
605
  try {
535
606
  const parsed = JSON.parse(text);
@@ -603,18 +674,127 @@ function originOf(url) {
603
674
  * Decode a response body by the charset of its Content-Type (UTF-8 when none is
604
675
  * given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
605
676
  * that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
606
- * sends UTF-8; this matters for proxies and mirrors that re-encode.
677
+ * sends UTF-8; this matters for proxies and mirrors that re-encode. An unknown charset
678
+ * label is a CkanParseError naming `request`. Bytes that are not valid in the charset become
679
+ * U+FFFD, and `replaced` counts them: exactly for UTF-8 and the single-byte charsets; for
680
+ * UTF-16 a U+FFFD the server sent counts too.
607
681
  */
608
- function decodeBody(body, contentType, url) {
682
+ function decodeBody(body, contentType, request) {
609
683
  const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
610
684
  let decoder;
611
685
  try {
612
- decoder = new TextDecoder(charset);
686
+ decoder = new TextDecoder(charset, { fatal: true });
613
687
  }
614
688
  catch {
615
- throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
689
+ throw new CkanParseError(`${request}: unsupported response charset "${cutForMessage(sanitizeServerText(charset))}".`);
616
690
  }
617
- return decoder.decode(body);
691
+ try {
692
+ return { text: decoder.decode(body), replaced: 0 };
693
+ }
694
+ catch (err) {
695
+ if (isStringTooLong(err))
696
+ throw tooLargeForText(request, body);
697
+ // Not valid in the charset: decode with replacement, and count what was replaced.
698
+ let text;
699
+ try {
700
+ text = new TextDecoder(decoder.encoding).decode(body);
701
+ }
702
+ catch (again) {
703
+ if (isStringTooLong(again))
704
+ throw tooLargeForText(request, body);
705
+ throw again;
706
+ }
707
+ const sent = decoder.encoding === "utf-8" ? countBytes(body, REPLACEMENT_UTF8) : 0;
708
+ return { text, replaced: countChar(text, 0xfffd) - sent };
709
+ }
710
+ }
711
+ /**
712
+ * True for V8's error for a string past its longest one (2^29 − 24 characters on 64-bit): a body
713
+ * of 512 MB and more, possible once `maxResponseBytes` is lifted, can't be decoded into one.
714
+ */
715
+ function isStringTooLong(err) {
716
+ return err?.code === "ERR_STRING_TOO_LONG";
717
+ }
718
+ /** The parse error for a body too large to decode: it used to escape as a plain Error ("Unexpected error"). */
719
+ function tooLargeForText(request, body) {
720
+ return new CkanParseError(`${request}: the answer is too large to read as text (${body.byteLength} bytes; the longest string is ${MAX_STRING_LENGTH} characters).`);
721
+ }
722
+ /**
723
+ * The body of an error answer as text, kept on the error (`body`). One too large to be a string
724
+ * (maxResponseBytes lifted) is left out: the status is the error.
725
+ */
726
+ function errorBodyText(body) {
727
+ try {
728
+ return body.toString("utf8");
729
+ }
730
+ catch (err) {
731
+ if (!isStringTooLong(err))
732
+ throw err;
733
+ return "";
734
+ }
735
+ }
736
+ /** U+FFFD in UTF-8. */
737
+ const REPLACEMENT_UTF8 = Buffer.from([0xef, 0xbf, 0xbd]);
738
+ /** How often `needle` occurs in `body`. */
739
+ function countBytes(body, needle) {
740
+ let n = 0;
741
+ for (let at = body.indexOf(needle); at !== -1; at = body.indexOf(needle, at + needle.length))
742
+ n++;
743
+ return n;
744
+ }
745
+ /** How often the UTF-16 unit `code` occurs in `text`. */
746
+ function countChar(text, code) {
747
+ let n = 0;
748
+ for (let i = 0; i < text.length; i++)
749
+ if (text.charCodeAt(i) === code)
750
+ n++;
751
+ return n;
752
+ }
753
+ /** A JSON escape of a surrogate, `\uD800`–`\uDFFF`, anywhere in a text (a cheap pre-check). */
754
+ const SURROGATE_ESCAPE = /\\u[dD][89a-fA-F][0-9a-fA-F]{2}/;
755
+ /** The value of the four hex digits of a `\uXXXX` escape whose `u` is at `at`, or -1. */
756
+ function escapeAt(json, at) {
757
+ if (json.charCodeAt(at) !== 0x75)
758
+ return -1;
759
+ const hex = json.slice(at + 1, at + 5);
760
+ return /^[0-9a-fA-F]{4}$/.test(hex) ? parseInt(hex, 16) : -1;
761
+ }
762
+ /**
763
+ * `json` (decoded answer text) with every lone surrogate written as a JSON escape (`\ud800` not
764
+ * followed by a low-surrogate escape, or a `\udc00` not preceded by a high one) rewritten as
765
+ * `\ufffd`, and how many. JSON.parse would make such an escape half a character, which a JSON
766
+ * printer writes back as `\ud800` and `jq` refuses (the whole output with it). A raw lone
767
+ * surrogate cannot occur: TextDecoder never makes one. One linear pass, and only when the text
768
+ * holds a surrogate escape at all; the text is copied only when one is lone.
769
+ */
770
+ function wellFormedEscapes(json) {
771
+ if (!SURROGATE_ESCAPE.test(json))
772
+ return { json, replaced: 0 };
773
+ let out = "";
774
+ let copied = 0;
775
+ let replaced = 0;
776
+ for (let i = 0; i < json.length; i++) {
777
+ if (json.charCodeAt(i) !== 0x5c)
778
+ continue;
779
+ const code = escapeAt(json, i + 1);
780
+ if (code < 0xd800 || code > 0xdfff) {
781
+ i++; // an escape of another kind (\", \\, \n, \u0041): skip its second character
782
+ continue;
783
+ }
784
+ const pairedLow = code <= 0xdbff && json.charCodeAt(i + 6) === 0x5c && isLowSurrogate(escapeAt(json, i + 7));
785
+ if (pairedLow) {
786
+ i += 11;
787
+ continue;
788
+ }
789
+ out += json.slice(copied, i) + "\\ufffd";
790
+ copied = i + 6;
791
+ replaced++;
792
+ i += 5;
793
+ }
794
+ return replaced === 0 ? { json, replaced } : { json: out + json.slice(copied), replaced };
795
+ }
796
+ function isLowSurrogate(code) {
797
+ return code >= 0xdc00 && code <= 0xdfff;
618
798
  }
619
799
  /** Resolve a Location header against the current URL; undefined if missing or malformed. */
620
800
  function resolveLocation(location, base) {
@@ -627,3 +807,139 @@ function resolveLocation(location, base) {
627
807
  return undefined;
628
808
  }
629
809
  }
810
+ /** A JSON number token, matched at a given position (sticky). */
811
+ const JSON_NUMBER = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
812
+ /** The smallest positive normal double, 2^-1022. Below it a double has fewer significant digits. */
813
+ const MIN_NORMAL = 2.2250738585072014e-308;
814
+ /**
815
+ * Whether the double `value` (finite) prints as the number `literal` wrote, checked where a
816
+ * double can be off in its leading digits: past 2^53, where every double is an integer, at 0,
817
+ * which a literal too small for a double becomes, and below the smallest normal double, where a
818
+ * subnormal has fewer significant digits (`3e-324` becomes `5e-324`). There the CLI prints
819
+ * `String(value)`, the shortest decimal that reads back as the same double: `12345678901234567890`
820
+ * prints as `12345678901234567000`, a changed value, while `1E23`, `1.7976931348623157E308` or
821
+ * `5e-324` (a double a server printed in its own shortest form) print as the same number.
822
+ */
823
+ function exactAtTheEdges(literal, value) {
824
+ const size = Math.abs(value);
825
+ if (value !== 0 && size <= Number.MAX_SAFE_INTEGER && size >= MIN_NORMAL)
826
+ return true;
827
+ const written = decimalOf(literal);
828
+ if (value === 0)
829
+ return written.digits === "";
830
+ const printed = decimalOf(String(value));
831
+ return written.negative === printed.negative && written.digits === printed.digits && written.exponent === printed.exponent;
832
+ }
833
+ /** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
834
+ function decimalOf(text) {
835
+ const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
836
+ const all = (whole + fraction).replace(/^0+/, "");
837
+ const digits = all.replace(/0+$/, "");
838
+ return { negative: sign === "-", digits, exponent: Number(exp) - fraction.length + (all.length - digits.length) };
839
+ }
840
+ /** A JSON number token as the grammar allows it, matched at a given position (sticky). */
841
+ const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
842
+ /** Whether the member name `json` holds between `start` and `end` (its quotes) is one of `keys`, compared in place. */
843
+ function isDeclared(json, start, end, keys) {
844
+ for (const key of keys) {
845
+ if (end - start - 1 === key.length && json.startsWith(key, start + 1))
846
+ return true;
847
+ }
848
+ return false;
849
+ }
850
+ /** The index of the first character at or after `i` that is not JSON whitespace. */
851
+ function skipSpace(json, i) {
852
+ for (; i < json.length; i++) {
853
+ const c = json.charCodeAt(i);
854
+ if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d)
855
+ break;
856
+ }
857
+ return i;
858
+ }
859
+ /**
860
+ * One linear pass over `json` (the text of an answer) that skips strings and looks at every
861
+ * number literal:
862
+ * - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
863
+ * 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
864
+ * it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
865
+ * so a malformed one (`01`) still fails JSON.parse;
866
+ * - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
867
+ * cannot carry exactly is reported in `inexact`.
868
+ * Nothing is allocated per value: a member name is compared in place (only a name written with
869
+ * an escape is decoded), and the text is copied only when a declared member is found. No
870
+ * reviver, which would recurse per nesting level and overflow on deep data.
871
+ */
872
+ export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
873
+ const keys = bigIntegerKeys.size === 0 ? undefined : [...bigIntegerKeys];
874
+ let inexact;
875
+ let out;
876
+ let copied = 0;
877
+ for (let i = 0; i < json.length; i++) {
878
+ const c = json.charCodeAt(i);
879
+ if (c === 0x22) {
880
+ // a string: skip to its closing quote, past escapes
881
+ const start = i;
882
+ let escaped = false;
883
+ for (i++; i < json.length; i++) {
884
+ const d = json.charCodeAt(i);
885
+ if (d === 0x5c) {
886
+ escaped = true;
887
+ i++;
888
+ }
889
+ else if (d === 0x22)
890
+ break;
891
+ }
892
+ if (keys === undefined)
893
+ continue;
894
+ // A member name is a string followed by a colon; is it a declared one with a number value?
895
+ const colon = skipSpace(json, i + 1);
896
+ if (json.charCodeAt(colon) !== 0x3a)
897
+ continue;
898
+ if (!(escaped ? bigIntegerKeys.has(decodedName(json.slice(start, i + 1))) : isDeclared(json, start, i, keys)))
899
+ continue;
900
+ const at = skipSpace(json, colon + 1);
901
+ STRICT_JSON_NUMBER.lastIndex = at;
902
+ const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
903
+ if (literal === undefined)
904
+ continue;
905
+ out = (out ?? "") + json.slice(copied, at) + '"' + literal + '"';
906
+ copied = at + literal.length;
907
+ i = copied - 1;
908
+ }
909
+ else if (c === 0x2d || (c >= 0x30 && c <= 0x39)) {
910
+ JSON_NUMBER.lastIndex = i;
911
+ const literal = JSON_NUMBER.exec(json)?.[0];
912
+ if (literal === undefined)
913
+ continue;
914
+ const value = Number(literal);
915
+ if (inexact === undefined && (!Number.isFinite(value) || !exactAtTheEdges(literal, value))) {
916
+ inexact = literal;
917
+ if (keys === undefined)
918
+ break;
919
+ }
920
+ i += literal.length - 1;
921
+ }
922
+ }
923
+ return { json: out === undefined ? json : out + json.slice(copied), inexact };
924
+ }
925
+ /** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
926
+ function decodedName(quoted) {
927
+ try {
928
+ return String(JSON.parse(quoted));
929
+ }
930
+ catch {
931
+ return "";
932
+ }
933
+ }
934
+ /**
935
+ * The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
936
+ * undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
937
+ * 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
938
+ * `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
939
+ * (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
940
+ * `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
941
+ * precision, as JSON means it. {@link scanNumbers} without declared members.
942
+ */
943
+ export function firstInexactNumber(json) {
944
+ return scanNumbers(json).inexact;
945
+ }
@@ -51,6 +51,18 @@ export declare function cutForMessage(text: string): string;
51
51
  * Text no longer than `max` is returned as it is; the caller marks a cut.
52
52
  */
53
53
  export declare function cutText(text: string, max: number): string;
54
+ /**
55
+ * Quote a value for an error message, as `JSON.stringify` does, and also escape what it
56
+ * leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
57
+ * format character (bidi overrides, zero-width characters) and the line and paragraph
58
+ * separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
59
+ * messages, and it can come from a pipeline or from the portal; an override in it would
60
+ * reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
61
+ * was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
62
+ * "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
63
+ * The quoted part stays a valid JSON string.
64
+ */
65
+ export declare function quoteValue(value: string): string;
54
66
  /**
55
67
  * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
56
68
  * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
@@ -96,16 +108,19 @@ export declare class CkanApiError extends CkanError {
96
108
  /**
97
109
  * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
98
110
  * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
99
- * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
111
+ * message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
100
112
  * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
101
113
  * or undefined when the error carries none.
102
114
  */
103
115
  export declare class CkanActionError extends CkanError {
104
116
  readonly action: string;
105
117
  readonly errorType: string | undefined;
118
+ /** The URL that answered, userinfo redacted; empty when the error was built without one. */
119
+ readonly url: string;
106
120
  constructor(message: string, args: {
107
121
  action: string;
108
122
  errorType?: string | undefined;
123
+ url?: string | undefined;
109
124
  });
110
125
  /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
111
126
  get isNotFound(): boolean;
@@ -134,6 +134,35 @@ export function cutText(text, max) {
134
134
  function isHighSurrogate(c) {
135
135
  return c >= 0xd800 && c <= 0xdbff;
136
136
  }
137
+ /** Unicode General Category Cf (format): bidi controls, zero-width characters, U+00AD, U+FEFF, … */
138
+ const FORMAT_CHAR = /\p{Cf}/u;
139
+ /**
140
+ * Quote a value for an error message, as `JSON.stringify` does, and also escape what it
141
+ * leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
142
+ * format character (bidi overrides, zero-width characters) and the line and paragraph
143
+ * separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
144
+ * messages, and it can come from a pipeline or from the portal; an override in it would
145
+ * reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
146
+ * was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
147
+ * "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
148
+ * The quoted part stays a valid JSON string.
149
+ */
150
+ export function quoteValue(value) {
151
+ const cut = value.length > MAX_MESSAGE_VALUE_LENGTH;
152
+ let out = "";
153
+ for (const ch of JSON.stringify(cut ? cutText(value, MAX_MESSAGE_VALUE_LENGTH) : value)) {
154
+ const n = ch.codePointAt(0) ?? 0;
155
+ if ((n >= 0x7f && n <= 0x9f) || n === 0x2028 || n === 0x2029 || FORMAT_CHAR.test(ch)) {
156
+ // One escape per UTF-16 unit, so a character above U+FFFF stays valid JSON.
157
+ for (let i = 0; i < ch.length; i++)
158
+ out += `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`;
159
+ }
160
+ else {
161
+ out += ch;
162
+ }
163
+ }
164
+ return cut ? `${out}…` : out;
165
+ }
137
166
  /**
138
167
  * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
139
168
  * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
@@ -201,17 +230,20 @@ export class CkanApiError extends CkanError {
201
230
  /**
202
231
  * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
203
232
  * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
204
- * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
233
+ * message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
205
234
  * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
206
235
  * or undefined when the error carries none.
207
236
  */
208
237
  export class CkanActionError extends CkanError {
209
238
  action;
210
239
  errorType;
240
+ /** The URL that answered, userinfo redacted; empty when the error was built without one. */
241
+ url;
211
242
  constructor(message, args) {
212
243
  super(message);
213
244
  this.action = args.action;
214
245
  this.errorType = args.errorType;
246
+ this.url = args.url === undefined ? "" : redactUrl(args.url);
215
247
  }
216
248
  /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
217
249
  get isNotFound() {
@@ -5,8 +5,8 @@ export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
- export { assertValid } from "./validate.js";
8
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
+ export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
12
12
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -3,8 +3,8 @@ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot }
3
3
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
- export { assertValid } from "./validate.js";
6
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
+ export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
10
10
  export * from "./types.js";
@@ -12,7 +12,10 @@ export declare function findPortal(idOrUrl: string, list: readonly Portal[]): Po
12
12
  /** The outcome of `checkPortal`. */
13
13
  export interface PortalCheck {
14
14
  working: boolean;
15
- /** A short reason when not working, e.g. `HTTP 404`, `host not found`. */
15
+ /**
16
+ * A short reason when not working, e.g. `HTTP 404`, `host not found`, `not JSON (text/html)`,
17
+ * `invalid JSON`, `inexact number` (valid JSON with a number a double can't carry), `empty answer`.
18
+ */
16
19
  problem: string | null;
17
20
  datasets: number | null;
18
21
  /** From `status_show`, which some portals lock (Berlin): then null. */
@@ -40,18 +43,18 @@ export declare const DEFAULT_CHECK_CONCURRENCY = 6;
40
43
  /** Options for `checkPortalUrls` and `checkPortals`. */
41
44
  export interface CheckPortalsOptions {
42
45
  /** Client options for every check; `baseUrl` is replaced by each portal's URL. */
43
- engineOptions?: EngineOptions;
46
+ engineOptions?: EngineOptions | undefined;
44
47
  /** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
45
- createClient?: (options: EngineOptions) => CkanClient;
48
+ createClient?: ((options: EngineOptions) => CkanClient) | undefined;
46
49
  /** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
47
- concurrency?: number;
50
+ concurrency?: number | undefined;
48
51
  /**
49
52
  * When set, a failed check is repeated once after this pause (ms), so one
50
53
  * timeout does not count as down. Unset (the default): a single try.
51
54
  */
52
- retryDelayMs?: number;
55
+ retryDelayMs?: number | undefined;
53
56
  /** Injectable sleep for the retry pause, for tests. */
54
- sleep?: (ms: number) => Promise<void>;
57
+ sleep?: ((ms: number) => Promise<void>) | undefined;
55
58
  }
56
59
  /**
57
60
  * Check the CKAN portals at `urls` live (`checkPortal`), with at most
@@ -66,5 +69,5 @@ export declare function checkPortalUrls(urls: readonly string[], options?: Check
66
69
  * What `ckan portals --check` prints.
67
70
  */
68
71
  export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
69
- date?: string;
72
+ date?: string | undefined;
70
73
  }): Promise<Portal[]>;
@@ -94,13 +94,21 @@ function problemOf(err) {
94
94
  }
95
95
  const message = err instanceof Error ? err.message : String(err);
96
96
  if (err instanceof CkanParseError) {
97
- const got = /but got (.+)$/.exec(message);
97
+ // The engine's "GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "<type>")."
98
+ const got = /\(expected JSON, got Content-Type "(.*)"\)\.$/.exec(message);
98
99
  if (got)
99
100
  return `not JSON (${got[1]})`;
100
- if (/not a CKAN Action API response/.test(message))
101
+ if (/: the answer is not a CKAN Action API response; /.test(message))
101
102
  return "not a CKAN Action API";
102
- if (/^Unexpected response shape/.test(message))
103
+ if (/: unexpected response shape: /.test(message))
103
104
  return "unexpected response shape";
105
+ // Valid JSON whose number a double can't carry, or no body at all: not "invalid JSON".
106
+ if (/: the number .* cannot be represented exactly\.$/.test(message))
107
+ return "inexact number";
108
+ if (/: empty response body\.$/.test(message))
109
+ return "empty answer";
110
+ if (/: unsupported response charset /.test(message))
111
+ return "unsupported charset";
104
112
  return "invalid JSON";
105
113
  }
106
114
  if (err instanceof CkanNetworkError) {