@maschinenlesbar.org/ckan-cli 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +80 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +289 -41
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +16 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +33 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +64 -9
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +169 -18
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.js +76 -6
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
+
import { constants } from "node:buffer";
|
|
4
5
|
import { TextDecoder } from "node:util";
|
|
5
6
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
6
7
|
import { buildQueryString } from "./query.js";
|
|
7
8
|
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
9
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
10
|
+
// quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
|
|
11
|
+
export { quoteValue } from "./errors.js";
|
|
9
12
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
13
|
+
/** V8's longest string (2^29 − 24 characters on 64-bit). */
|
|
14
|
+
const MAX_STRING_LENGTH = constants.MAX_STRING_LENGTH;
|
|
10
15
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
11
16
|
const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
12
17
|
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
@@ -200,14 +205,16 @@ function bodyBytes(value) {
|
|
|
200
205
|
*/
|
|
201
206
|
function plainHeaders(headers) {
|
|
202
207
|
const h = headers;
|
|
208
|
+
// Records built from an answer have no prototype: a header named `__proto__` is an own
|
|
209
|
+
// entry like any other, never the record's prototype (whose entries would read as headers).
|
|
203
210
|
if (typeof h.get === "function" && typeof h.forEach === "function") {
|
|
204
|
-
const record =
|
|
211
|
+
const record = Object.create(null);
|
|
205
212
|
h.forEach.call(headers, (value, name) => {
|
|
206
213
|
record[String(name).toLowerCase()] = value;
|
|
207
214
|
});
|
|
208
215
|
return record;
|
|
209
216
|
}
|
|
210
|
-
const record =
|
|
217
|
+
const record = Object.create(null);
|
|
211
218
|
for (const [name, value] of Object.entries(headers)) {
|
|
212
219
|
record[name.toLowerCase()] = value;
|
|
213
220
|
}
|
|
@@ -218,6 +225,22 @@ function headerValue(value) {
|
|
|
218
225
|
return Array.isArray(value) ? value[0] : value;
|
|
219
226
|
}
|
|
220
227
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
228
|
+
/** No declared big-integer members (`EngineOptions.bigIntegerKeys` left out). */
|
|
229
|
+
const NO_KEYS = new Set();
|
|
230
|
+
/**
|
|
231
|
+
* Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
|
|
232
|
+
* copied; anything else throws an CkanValidationError.
|
|
233
|
+
*/
|
|
234
|
+
export function bigIntegerKeysOption(value) {
|
|
235
|
+
if (value === undefined)
|
|
236
|
+
return NO_KEYS;
|
|
237
|
+
const iterable = typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
|
|
238
|
+
const names = iterable ? [...value] : [];
|
|
239
|
+
if (!iterable || names.some((key) => typeof key !== "string" || key === "")) {
|
|
240
|
+
throw new CkanValidationError("Invalid option bigIntegerKeys: expected a set of non-empty member names.");
|
|
241
|
+
}
|
|
242
|
+
return new Set(names);
|
|
243
|
+
}
|
|
221
244
|
export class RequestEngine {
|
|
222
245
|
// A real private field (not TypeScript's `private`): util.inspect, console.log and
|
|
223
246
|
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
@@ -240,6 +263,9 @@ export class RequestEngine {
|
|
|
240
263
|
maxResponseBytes;
|
|
241
264
|
sleep;
|
|
242
265
|
onRetry;
|
|
266
|
+
onWarning;
|
|
267
|
+
/** `EngineOptions.bigIntegerKeys`, copied. */
|
|
268
|
+
bigIntegerKeys;
|
|
243
269
|
constructor(options = {}) {
|
|
244
270
|
// The raw value, checked before the slash strip (the engine glues it into
|
|
245
271
|
// every URL), and here rather than only in the default transport: a library
|
|
@@ -273,6 +299,18 @@ export class RequestEngine {
|
|
|
273
299
|
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
274
300
|
this.onRetry =
|
|
275
301
|
options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
|
|
302
|
+
this.onWarning =
|
|
303
|
+
options.onWarning === undefined ? undefined : functionOption("onWarning", options.onWarning, () => { });
|
|
304
|
+
this.bigIntegerKeys = bigIntegerKeysOption(options.bigIntegerKeys);
|
|
305
|
+
}
|
|
306
|
+
/** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
|
|
307
|
+
warn(message) {
|
|
308
|
+
try {
|
|
309
|
+
this.onWarning?.(message);
|
|
310
|
+
}
|
|
311
|
+
catch {
|
|
312
|
+
// a logging hook is no reason to fail the request
|
|
313
|
+
}
|
|
276
314
|
}
|
|
277
315
|
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
278
316
|
async backOff(attempt, delayMs, url, status) {
|
|
@@ -499,7 +537,7 @@ export class RequestEngine {
|
|
|
499
537
|
: location
|
|
500
538
|
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
501
539
|
: "redirect not followed (no Location header)";
|
|
502
|
-
throw new CkanApiError({ status, url, method, body: this.scrub(body
|
|
540
|
+
throw new CkanApiError({ status, url, method, body: this.scrub(errorBodyText(body)), detail });
|
|
503
541
|
}
|
|
504
542
|
throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
505
543
|
retries: attempt,
|
|
@@ -509,35 +547,60 @@ export class RequestEngine {
|
|
|
509
547
|
return { data: body, contentType, status, url };
|
|
510
548
|
}
|
|
511
549
|
}
|
|
512
|
-
/** Perform a GET expecting JSON and parse it into `T
|
|
550
|
+
/** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
|
|
513
551
|
async getJson(path, query) {
|
|
552
|
+
return (await this.getAnswer(path, query)).value;
|
|
553
|
+
}
|
|
554
|
+
/**
|
|
555
|
+
* Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
|
|
556
|
+
* userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
|
|
557
|
+
* MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
|
|
558
|
+
* (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
|
|
559
|
+
* one failed; the client prefixes its own envelope and shape errors with it too.
|
|
560
|
+
*
|
|
561
|
+
* The text is never changed silently: bytes that are not valid in the charset, and lone
|
|
562
|
+
* surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
|
|
563
|
+
* well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
|
|
564
|
+
* `onWarning` is told how many. One bad character never sinks a whole answer.
|
|
565
|
+
*/
|
|
566
|
+
async getAnswer(path, query) {
|
|
514
567
|
const res = await this.request("GET", path, { query, accept: "application/json" });
|
|
515
|
-
const
|
|
568
|
+
const url = redactUrl(res.url);
|
|
569
|
+
const request = `GET ${cutForMessage(url)}`;
|
|
570
|
+
const decoded = decodeBody(res.data, res.contentType, request);
|
|
571
|
+
if (decoded.text.trim() === "") {
|
|
572
|
+
// A broken answer, not "no results".
|
|
573
|
+
throw new CkanParseError(`${request}: empty response body.`);
|
|
574
|
+
}
|
|
575
|
+
const repaired = wellFormedEscapes(decoded.text);
|
|
576
|
+
// Declared big integers are quoted first, so JSON.parse delivers them exactly.
|
|
577
|
+
const { json, inexact } = scanNumbers(repaired.json, this.bigIntegerKeys);
|
|
516
578
|
let parsed;
|
|
517
579
|
try {
|
|
518
|
-
parsed = JSON.parse(
|
|
580
|
+
parsed = JSON.parse(json);
|
|
519
581
|
}
|
|
520
582
|
catch (cause) {
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
|
|
525
|
-
const
|
|
526
|
-
|
|
527
|
-
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
528
|
-
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
529
|
-
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
583
|
+
// A site that is not a CKAN (or not at this path) typically answers with an HTML page and
|
|
584
|
+
// HTTP 200: name its type. The Content-Type is server text (latin-1 decoded, so a 0x9B byte
|
|
585
|
+
// is the 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
586
|
+
const type = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
|
|
587
|
+
const hint = type !== "" && !/json/i.test(type) ? ` (expected JSON, got Content-Type "${type}")` : "";
|
|
588
|
+
throw new CkanParseError(`${request}: the answer is not valid JSON${hint}.`, { cause: this.scrubCause(cause) });
|
|
530
589
|
}
|
|
531
590
|
// JSON.parse rounds and overflows numbers silently: refuse the answer rather than print a
|
|
532
591
|
// changed value.
|
|
533
|
-
const inexact = firstInexactNumber(text);
|
|
534
592
|
if (inexact !== undefined) {
|
|
535
|
-
throw new CkanParseError(
|
|
593
|
+
throw new CkanParseError(`${request}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
|
|
536
594
|
}
|
|
537
|
-
|
|
595
|
+
const replaced = decoded.replaced + repaired.replaced;
|
|
596
|
+
if (replaced > 0) {
|
|
597
|
+
this.warn(`${request}: ${replaced} ${replaced === 1 ? "character" : "characters"} of the answer ${replaced === 1 ? "was" : "were"} ` +
|
|
598
|
+
"not valid text (invalid bytes or a lone surrogate) and replaced with U+FFFD.");
|
|
599
|
+
}
|
|
600
|
+
return { value: parsed, url, request };
|
|
538
601
|
}
|
|
539
602
|
toApiError(method, url, status, body, hint, retry) {
|
|
540
|
-
const text = this.scrub(body
|
|
603
|
+
const text = this.scrub(errorBodyText(body));
|
|
541
604
|
let detail;
|
|
542
605
|
try {
|
|
543
606
|
const parsed = JSON.parse(text);
|
|
@@ -611,18 +674,127 @@ function originOf(url) {
|
|
|
611
674
|
* Decode a response body by the charset of its Content-Type (UTF-8 when none is
|
|
612
675
|
* given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
|
|
613
676
|
* that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
|
|
614
|
-
* sends UTF-8; this matters for proxies and mirrors that re-encode.
|
|
677
|
+
* sends UTF-8; this matters for proxies and mirrors that re-encode. An unknown charset
|
|
678
|
+
* label is a CkanParseError naming `request`. Bytes that are not valid in the charset become
|
|
679
|
+
* U+FFFD, and `replaced` counts them: exactly for UTF-8 and the single-byte charsets; for
|
|
680
|
+
* UTF-16 a U+FFFD the server sent counts too.
|
|
615
681
|
*/
|
|
616
|
-
function decodeBody(body, contentType,
|
|
682
|
+
function decodeBody(body, contentType, request) {
|
|
617
683
|
const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
|
|
618
684
|
let decoder;
|
|
619
685
|
try {
|
|
620
|
-
decoder = new TextDecoder(charset);
|
|
686
|
+
decoder = new TextDecoder(charset, { fatal: true });
|
|
621
687
|
}
|
|
622
688
|
catch {
|
|
623
|
-
throw new CkanParseError(
|
|
689
|
+
throw new CkanParseError(`${request}: unsupported response charset "${cutForMessage(sanitizeServerText(charset))}".`);
|
|
690
|
+
}
|
|
691
|
+
try {
|
|
692
|
+
return { text: decoder.decode(body), replaced: 0 };
|
|
693
|
+
}
|
|
694
|
+
catch (err) {
|
|
695
|
+
if (isStringTooLong(err))
|
|
696
|
+
throw tooLargeForText(request, body);
|
|
697
|
+
// Not valid in the charset: decode with replacement, and count what was replaced.
|
|
698
|
+
let text;
|
|
699
|
+
try {
|
|
700
|
+
text = new TextDecoder(decoder.encoding).decode(body);
|
|
701
|
+
}
|
|
702
|
+
catch (again) {
|
|
703
|
+
if (isStringTooLong(again))
|
|
704
|
+
throw tooLargeForText(request, body);
|
|
705
|
+
throw again;
|
|
706
|
+
}
|
|
707
|
+
const sent = decoder.encoding === "utf-8" ? countBytes(body, REPLACEMENT_UTF8) : 0;
|
|
708
|
+
return { text, replaced: countChar(text, 0xfffd) - sent };
|
|
624
709
|
}
|
|
625
|
-
|
|
710
|
+
}
|
|
711
|
+
/**
|
|
712
|
+
* True for V8's error for a string past its longest one (2^29 − 24 characters on 64-bit): a body
|
|
713
|
+
* of 512 MB and more, possible once `maxResponseBytes` is lifted, can't be decoded into one.
|
|
714
|
+
*/
|
|
715
|
+
function isStringTooLong(err) {
|
|
716
|
+
return err?.code === "ERR_STRING_TOO_LONG";
|
|
717
|
+
}
|
|
718
|
+
/** The parse error for a body too large to decode: it used to escape as a plain Error ("Unexpected error"). */
|
|
719
|
+
function tooLargeForText(request, body) {
|
|
720
|
+
return new CkanParseError(`${request}: the answer is too large to read as text (${body.byteLength} bytes; the longest string is ${MAX_STRING_LENGTH} characters).`);
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* The body of an error answer as text, kept on the error (`body`). One too large to be a string
|
|
724
|
+
* (maxResponseBytes lifted) is left out: the status is the error.
|
|
725
|
+
*/
|
|
726
|
+
function errorBodyText(body) {
|
|
727
|
+
try {
|
|
728
|
+
return body.toString("utf8");
|
|
729
|
+
}
|
|
730
|
+
catch (err) {
|
|
731
|
+
if (!isStringTooLong(err))
|
|
732
|
+
throw err;
|
|
733
|
+
return "";
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
/** U+FFFD in UTF-8. */
|
|
737
|
+
const REPLACEMENT_UTF8 = Buffer.from([0xef, 0xbf, 0xbd]);
|
|
738
|
+
/** How often `needle` occurs in `body`. */
|
|
739
|
+
function countBytes(body, needle) {
|
|
740
|
+
let n = 0;
|
|
741
|
+
for (let at = body.indexOf(needle); at !== -1; at = body.indexOf(needle, at + needle.length))
|
|
742
|
+
n++;
|
|
743
|
+
return n;
|
|
744
|
+
}
|
|
745
|
+
/** How often the UTF-16 unit `code` occurs in `text`. */
|
|
746
|
+
function countChar(text, code) {
|
|
747
|
+
let n = 0;
|
|
748
|
+
for (let i = 0; i < text.length; i++)
|
|
749
|
+
if (text.charCodeAt(i) === code)
|
|
750
|
+
n++;
|
|
751
|
+
return n;
|
|
752
|
+
}
|
|
753
|
+
/** A JSON escape of a surrogate, `\uD800`–`\uDFFF`, anywhere in a text (a cheap pre-check). */
|
|
754
|
+
const SURROGATE_ESCAPE = /\\u[dD][89a-fA-F][0-9a-fA-F]{2}/;
|
|
755
|
+
/** The value of the four hex digits of a `\uXXXX` escape whose `u` is at `at`, or -1. */
|
|
756
|
+
function escapeAt(json, at) {
|
|
757
|
+
if (json.charCodeAt(at) !== 0x75)
|
|
758
|
+
return -1;
|
|
759
|
+
const hex = json.slice(at + 1, at + 5);
|
|
760
|
+
return /^[0-9a-fA-F]{4}$/.test(hex) ? parseInt(hex, 16) : -1;
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* `json` (decoded answer text) with every lone surrogate written as a JSON escape (`\ud800` not
|
|
764
|
+
* followed by a low-surrogate escape, or a `\udc00` not preceded by a high one) rewritten as
|
|
765
|
+
* `\ufffd`, and how many. JSON.parse would make such an escape half a character, which a JSON
|
|
766
|
+
* printer writes back as `\ud800` and `jq` refuses (the whole output with it). A raw lone
|
|
767
|
+
* surrogate cannot occur: TextDecoder never makes one. One linear pass, and only when the text
|
|
768
|
+
* holds a surrogate escape at all; the text is copied only when one is lone.
|
|
769
|
+
*/
|
|
770
|
+
function wellFormedEscapes(json) {
|
|
771
|
+
if (!SURROGATE_ESCAPE.test(json))
|
|
772
|
+
return { json, replaced: 0 };
|
|
773
|
+
let out = "";
|
|
774
|
+
let copied = 0;
|
|
775
|
+
let replaced = 0;
|
|
776
|
+
for (let i = 0; i < json.length; i++) {
|
|
777
|
+
if (json.charCodeAt(i) !== 0x5c)
|
|
778
|
+
continue;
|
|
779
|
+
const code = escapeAt(json, i + 1);
|
|
780
|
+
if (code < 0xd800 || code > 0xdfff) {
|
|
781
|
+
i++; // an escape of another kind (\", \\, \n, \u0041): skip its second character
|
|
782
|
+
continue;
|
|
783
|
+
}
|
|
784
|
+
const pairedLow = code <= 0xdbff && json.charCodeAt(i + 6) === 0x5c && isLowSurrogate(escapeAt(json, i + 7));
|
|
785
|
+
if (pairedLow) {
|
|
786
|
+
i += 11;
|
|
787
|
+
continue;
|
|
788
|
+
}
|
|
789
|
+
out += json.slice(copied, i) + "\\ufffd";
|
|
790
|
+
copied = i + 6;
|
|
791
|
+
replaced++;
|
|
792
|
+
i += 5;
|
|
793
|
+
}
|
|
794
|
+
return replaced === 0 ? { json, replaced } : { json: out + json.slice(copied), replaced };
|
|
795
|
+
}
|
|
796
|
+
function isLowSurrogate(code) {
|
|
797
|
+
return code >= 0xdc00 && code <= 0xdfff;
|
|
626
798
|
}
|
|
627
799
|
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
628
800
|
function resolveLocation(location, base) {
|
|
@@ -637,16 +809,20 @@ function resolveLocation(location, base) {
|
|
|
637
809
|
}
|
|
638
810
|
/** A JSON number token, matched at a given position (sticky). */
|
|
639
811
|
const JSON_NUMBER = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
812
|
+
/** The smallest positive normal double, 2^-1022. Below it a double has fewer significant digits. */
|
|
813
|
+
const MIN_NORMAL = 2.2250738585072014e-308;
|
|
640
814
|
/**
|
|
641
815
|
* Whether the double `value` (finite) prints as the number `literal` wrote, checked where a
|
|
642
|
-
* double can be off
|
|
643
|
-
*
|
|
644
|
-
*
|
|
645
|
-
* `
|
|
646
|
-
*
|
|
816
|
+
* double can be off in its leading digits: past 2^53, where every double is an integer, at 0,
|
|
817
|
+
* which a literal too small for a double becomes, and below the smallest normal double, where a
|
|
818
|
+
* subnormal has fewer significant digits (`3e-324` becomes `5e-324`). There the CLI prints
|
|
819
|
+
* `String(value)`, the shortest decimal that reads back as the same double: `12345678901234567890`
|
|
820
|
+
* prints as `12345678901234567000`, a changed value, while `1E23`, `1.7976931348623157E308` or
|
|
821
|
+
* `5e-324` (a double a server printed in its own shortest form) print as the same number.
|
|
647
822
|
*/
|
|
648
823
|
function exactAtTheEdges(literal, value) {
|
|
649
|
-
|
|
824
|
+
const size = Math.abs(value);
|
|
825
|
+
if (value !== 0 && size <= Number.MAX_SAFE_INTEGER && size >= MIN_NORMAL)
|
|
650
826
|
return true;
|
|
651
827
|
const written = decimalOf(literal);
|
|
652
828
|
if (value === 0)
|
|
@@ -661,26 +837,74 @@ function decimalOf(text) {
|
|
|
661
837
|
const digits = all.replace(/0+$/, "");
|
|
662
838
|
return { negative: sign === "-", digits, exponent: Number(exp) - fraction.length + (all.length - digits.length) };
|
|
663
839
|
}
|
|
840
|
+
/** A JSON number token as the grammar allows it, matched at a given position (sticky). */
|
|
841
|
+
const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
842
|
+
/** Whether the member name `json` holds between `start` and `end` (its quotes) is one of `keys`, compared in place. */
|
|
843
|
+
function isDeclared(json, start, end, keys) {
|
|
844
|
+
for (const key of keys) {
|
|
845
|
+
if (end - start - 1 === key.length && json.startsWith(key, start + 1))
|
|
846
|
+
return true;
|
|
847
|
+
}
|
|
848
|
+
return false;
|
|
849
|
+
}
|
|
850
|
+
/** The index of the first character at or after `i` that is not JSON whitespace. */
|
|
851
|
+
function skipSpace(json, i) {
|
|
852
|
+
for (; i < json.length; i++) {
|
|
853
|
+
const c = json.charCodeAt(i);
|
|
854
|
+
if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d)
|
|
855
|
+
break;
|
|
856
|
+
}
|
|
857
|
+
return i;
|
|
858
|
+
}
|
|
664
859
|
/**
|
|
665
|
-
*
|
|
666
|
-
*
|
|
667
|
-
*
|
|
668
|
-
*
|
|
669
|
-
*
|
|
670
|
-
*
|
|
860
|
+
* One linear pass over `json` (the text of an answer) that skips strings and looks at every
|
|
861
|
+
* number literal:
|
|
862
|
+
* - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
|
|
863
|
+
* 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
|
|
864
|
+
* it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
|
|
865
|
+
* so a malformed one (`01`) still fails JSON.parse;
|
|
866
|
+
* - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
|
|
867
|
+
* cannot carry exactly is reported in `inexact`.
|
|
868
|
+
* Nothing is allocated per value: a member name is compared in place (only a name written with
|
|
869
|
+
* an escape is decoded), and the text is copied only when a declared member is found. No
|
|
870
|
+
* reviver, which would recurse per nesting level and overflow on deep data.
|
|
671
871
|
*/
|
|
672
|
-
function
|
|
872
|
+
export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
873
|
+
const keys = bigIntegerKeys.size === 0 ? undefined : [...bigIntegerKeys];
|
|
874
|
+
let inexact;
|
|
875
|
+
let out;
|
|
876
|
+
let copied = 0;
|
|
673
877
|
for (let i = 0; i < json.length; i++) {
|
|
674
878
|
const c = json.charCodeAt(i);
|
|
675
879
|
if (c === 0x22) {
|
|
676
880
|
// a string: skip to its closing quote, past escapes
|
|
881
|
+
const start = i;
|
|
882
|
+
let escaped = false;
|
|
677
883
|
for (i++; i < json.length; i++) {
|
|
678
884
|
const d = json.charCodeAt(i);
|
|
679
|
-
if (d === 0x5c)
|
|
885
|
+
if (d === 0x5c) {
|
|
886
|
+
escaped = true;
|
|
680
887
|
i++;
|
|
888
|
+
}
|
|
681
889
|
else if (d === 0x22)
|
|
682
890
|
break;
|
|
683
891
|
}
|
|
892
|
+
if (keys === undefined)
|
|
893
|
+
continue;
|
|
894
|
+
// A member name is a string followed by a colon; is it a declared one with a number value?
|
|
895
|
+
const colon = skipSpace(json, i + 1);
|
|
896
|
+
if (json.charCodeAt(colon) !== 0x3a)
|
|
897
|
+
continue;
|
|
898
|
+
if (!(escaped ? bigIntegerKeys.has(decodedName(json.slice(start, i + 1))) : isDeclared(json, start, i, keys)))
|
|
899
|
+
continue;
|
|
900
|
+
const at = skipSpace(json, colon + 1);
|
|
901
|
+
STRICT_JSON_NUMBER.lastIndex = at;
|
|
902
|
+
const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
|
|
903
|
+
if (literal === undefined)
|
|
904
|
+
continue;
|
|
905
|
+
out = (out ?? "") + json.slice(copied, at) + '"' + literal + '"';
|
|
906
|
+
copied = at + literal.length;
|
|
907
|
+
i = copied - 1;
|
|
684
908
|
}
|
|
685
909
|
else if (c === 0x2d || (c >= 0x30 && c <= 0x39)) {
|
|
686
910
|
JSON_NUMBER.lastIndex = i;
|
|
@@ -688,10 +912,34 @@ function firstInexactNumber(json) {
|
|
|
688
912
|
if (literal === undefined)
|
|
689
913
|
continue;
|
|
690
914
|
const value = Number(literal);
|
|
691
|
-
if (!Number.isFinite(value) || !exactAtTheEdges(literal, value))
|
|
692
|
-
|
|
915
|
+
if (inexact === undefined && (!Number.isFinite(value) || !exactAtTheEdges(literal, value))) {
|
|
916
|
+
inexact = literal;
|
|
917
|
+
if (keys === undefined)
|
|
918
|
+
break;
|
|
919
|
+
}
|
|
693
920
|
i += literal.length - 1;
|
|
694
921
|
}
|
|
695
922
|
}
|
|
696
|
-
return undefined;
|
|
923
|
+
return { json: out === undefined ? json : out + json.slice(copied), inexact };
|
|
924
|
+
}
|
|
925
|
+
/** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
|
|
926
|
+
function decodedName(quoted) {
|
|
927
|
+
try {
|
|
928
|
+
return String(JSON.parse(quoted));
|
|
929
|
+
}
|
|
930
|
+
catch {
|
|
931
|
+
return "";
|
|
932
|
+
}
|
|
933
|
+
}
|
|
934
|
+
/**
|
|
935
|
+
* The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
|
|
936
|
+
* undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
|
|
937
|
+
* 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
|
|
938
|
+
* `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
|
|
939
|
+
* (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
|
|
940
|
+
* `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
|
|
941
|
+
* precision, as JSON means it. {@link scanNumbers} without declared members.
|
|
942
|
+
*/
|
|
943
|
+
export function firstInexactNumber(json) {
|
|
944
|
+
return scanNumbers(json).inexact;
|
|
697
945
|
}
|
|
@@ -51,6 +51,18 @@ export declare function cutForMessage(text: string): string;
|
|
|
51
51
|
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
52
52
|
*/
|
|
53
53
|
export declare function cutText(text: string, max: number): string;
|
|
54
|
+
/**
|
|
55
|
+
* Quote a value for an error message, as `JSON.stringify` does, and also escape what it
|
|
56
|
+
* leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
|
|
57
|
+
* format character (bidi overrides, zero-width characters) and the line and paragraph
|
|
58
|
+
* separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
|
|
59
|
+
* messages, and it can come from a pipeline or from the portal; an override in it would
|
|
60
|
+
* reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
|
|
61
|
+
* was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
|
|
62
|
+
* "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
|
|
63
|
+
* The quoted part stays a valid JSON string.
|
|
64
|
+
*/
|
|
65
|
+
export declare function quoteValue(value: string): string;
|
|
54
66
|
/**
|
|
55
67
|
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
56
68
|
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
@@ -96,16 +108,19 @@ export declare class CkanApiError extends CkanError {
|
|
|
96
108
|
/**
|
|
97
109
|
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
98
110
|
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
99
|
-
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
111
|
+
* message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
100
112
|
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
101
113
|
* or undefined when the error carries none.
|
|
102
114
|
*/
|
|
103
115
|
export declare class CkanActionError extends CkanError {
|
|
104
116
|
readonly action: string;
|
|
105
117
|
readonly errorType: string | undefined;
|
|
118
|
+
/** The URL that answered, userinfo redacted; empty when the error was built without one. */
|
|
119
|
+
readonly url: string;
|
|
106
120
|
constructor(message: string, args: {
|
|
107
121
|
action: string;
|
|
108
122
|
errorType?: string | undefined;
|
|
123
|
+
url?: string | undefined;
|
|
109
124
|
});
|
|
110
125
|
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
111
126
|
get isNotFound(): boolean;
|
|
@@ -134,6 +134,35 @@ export function cutText(text, max) {
|
|
|
134
134
|
function isHighSurrogate(c) {
|
|
135
135
|
return c >= 0xd800 && c <= 0xdbff;
|
|
136
136
|
}
|
|
137
|
+
/** Unicode General Category Cf (format): bidi controls, zero-width characters, U+00AD, U+FEFF, … */
|
|
138
|
+
const FORMAT_CHAR = /\p{Cf}/u;
|
|
139
|
+
/**
|
|
140
|
+
* Quote a value for an error message, as `JSON.stringify` does, and also escape what it
|
|
141
|
+
* leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
|
|
142
|
+
* format character (bidi overrides, zero-width characters) and the line and paragraph
|
|
143
|
+
* separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
|
|
144
|
+
* messages, and it can come from a pipeline or from the portal; an override in it would
|
|
145
|
+
* reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
|
|
146
|
+
* was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
|
|
147
|
+
* "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
|
|
148
|
+
* The quoted part stays a valid JSON string.
|
|
149
|
+
*/
|
|
150
|
+
export function quoteValue(value) {
|
|
151
|
+
const cut = value.length > MAX_MESSAGE_VALUE_LENGTH;
|
|
152
|
+
let out = "";
|
|
153
|
+
for (const ch of JSON.stringify(cut ? cutText(value, MAX_MESSAGE_VALUE_LENGTH) : value)) {
|
|
154
|
+
const n = ch.codePointAt(0) ?? 0;
|
|
155
|
+
if ((n >= 0x7f && n <= 0x9f) || n === 0x2028 || n === 0x2029 || FORMAT_CHAR.test(ch)) {
|
|
156
|
+
// One escape per UTF-16 unit, so a character above U+FFFF stays valid JSON.
|
|
157
|
+
for (let i = 0; i < ch.length; i++)
|
|
158
|
+
out += `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`;
|
|
159
|
+
}
|
|
160
|
+
else {
|
|
161
|
+
out += ch;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
return cut ? `${out}…` : out;
|
|
165
|
+
}
|
|
137
166
|
/**
|
|
138
167
|
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
139
168
|
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
@@ -201,17 +230,20 @@ export class CkanApiError extends CkanError {
|
|
|
201
230
|
/**
|
|
202
231
|
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
203
232
|
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
204
|
-
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
233
|
+
* message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
205
234
|
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
206
235
|
* or undefined when the error carries none.
|
|
207
236
|
*/
|
|
208
237
|
export class CkanActionError extends CkanError {
|
|
209
238
|
action;
|
|
210
239
|
errorType;
|
|
240
|
+
/** The URL that answered, userinfo redacted; empty when the error was built without one. */
|
|
241
|
+
url;
|
|
211
242
|
constructor(message, args) {
|
|
212
243
|
super(message);
|
|
213
244
|
this.action = args.action;
|
|
214
245
|
this.errorType = args.errorType;
|
|
246
|
+
this.url = args.url === undefined ? "" : redactUrl(args.url);
|
|
215
247
|
}
|
|
216
248
|
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
217
249
|
get isNotFound() {
|
|
@@ -5,8 +5,8 @@ export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
-
export { assertValid, normalizeText } from "./validate.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
+
export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
|
12
12
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -3,8 +3,8 @@ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot }
|
|
|
3
3
|
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
-
export { assertValid, normalizeText } from "./validate.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
+
export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
10
10
|
export * from "./types.js";
|
|
@@ -12,7 +12,10 @@ export declare function findPortal(idOrUrl: string, list: readonly Portal[]): Po
|
|
|
12
12
|
/** The outcome of `checkPortal`. */
|
|
13
13
|
export interface PortalCheck {
|
|
14
14
|
working: boolean;
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* A short reason when not working, e.g. `HTTP 404`, `host not found`, `not JSON (text/html)`,
|
|
17
|
+
* `invalid JSON`, `inexact number` (valid JSON with a number a double can't carry), `empty answer`.
|
|
18
|
+
*/
|
|
16
19
|
problem: string | null;
|
|
17
20
|
datasets: number | null;
|
|
18
21
|
/** From `status_show`, which some portals lock (Berlin): then null. */
|
|
@@ -94,13 +94,21 @@ function problemOf(err) {
|
|
|
94
94
|
}
|
|
95
95
|
const message = err instanceof Error ? err.message : String(err);
|
|
96
96
|
if (err instanceof CkanParseError) {
|
|
97
|
-
|
|
97
|
+
// The engine's "GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "<type>")."
|
|
98
|
+
const got = /\(expected JSON, got Content-Type "(.*)"\)\.$/.exec(message);
|
|
98
99
|
if (got)
|
|
99
100
|
return `not JSON (${got[1]})`;
|
|
100
|
-
if (
|
|
101
|
+
if (/: the answer is not a CKAN Action API response; /.test(message))
|
|
101
102
|
return "not a CKAN Action API";
|
|
102
|
-
if (
|
|
103
|
+
if (/: unexpected response shape: /.test(message))
|
|
103
104
|
return "unexpected response shape";
|
|
105
|
+
// Valid JSON whose number a double can't carry, or no body at all: not "invalid JSON".
|
|
106
|
+
if (/: the number .* cannot be represented exactly\.$/.test(message))
|
|
107
|
+
return "inexact number";
|
|
108
|
+
if (/: empty response body\.$/.test(message))
|
|
109
|
+
return "empty answer";
|
|
110
|
+
if (/: unsupported response charset /.test(message))
|
|
111
|
+
return "unsupported charset";
|
|
104
112
|
return "invalid JSON";
|
|
105
113
|
}
|
|
106
114
|
if (err instanceof CkanNetworkError) {
|