@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +80 -17
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
+
import { constants } from "node:buffer";
|
|
4
5
|
import { TextDecoder } from "node:util";
|
|
5
6
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
6
7
|
import { buildQueryString } from "./query.js";
|
|
7
|
-
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, cutText, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, TextBuilder, } from "./errors.js";
|
|
8
9
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
10
|
+
// quoteValue lives with the other message helpers (validate.ts quotes refused values with it).
|
|
11
|
+
export { quoteValue } from "./errors.js";
|
|
9
12
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
13
|
+
/** V8's longest string (2^29 − 24 characters on 64-bit). */
|
|
14
|
+
const MAX_STRING_LENGTH = constants.MAX_STRING_LENGTH;
|
|
10
15
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
11
16
|
const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
12
17
|
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
@@ -75,35 +80,57 @@ export function isBidiControl(code) {
|
|
|
75
80
|
(code >= 0x202a && code <= 0x202e) ||
|
|
76
81
|
(code >= 0x2066 && code <= 0x2069));
|
|
77
82
|
}
|
|
83
|
+
/**
|
|
84
|
+
* The most server text (in UTF-16 units) `sanitizeServerText` reads; a longer text is cut first.
|
|
85
|
+
* Every message keeps far less of it (MAX_MESSAGE_VALUE_LENGTH, 500 characters), and cleaning all
|
|
86
|
+
* of a 16 MB error detail, one string node per character, exhausted a small heap.
|
|
87
|
+
*/
|
|
88
|
+
const MAX_SERVER_TEXT = 16 * 1024;
|
|
89
|
+
/**
|
|
90
|
+
* A whole terminal escape sequence: a CSI (`ESC [` or the 8-bit U+009B, parameters,
|
|
91
|
+
* intermediates, final byte), a terminated control string (OSC/DCS/SOS/PM/APC, `ESC ]` …
|
|
92
|
+
* up to BEL or ST, or their 8-bit forms) or a short `ESC` sequence. Dropping only the
|
|
93
|
+
* control bytes left the printable rest behind (`ESC ] 0 ; title BEL` became `]0;title`).
|
|
94
|
+
* An unterminated control string is left to the per-character filter, so it cannot
|
|
95
|
+
* swallow the rest of the message. A control string's body ends at an ESC or at another
|
|
96
|
+
* introducer: with the introducers allowed inside it, an unterminated run of them was matched
|
|
97
|
+
* from each one to the end of the text, quadratic (100 000 × U+009D took 16 s). Written with
|
|
98
|
+
* escapes, so the source holds no control bytes.
|
|
99
|
+
*/
|
|
100
|
+
const ESCAPE_SEQUENCE = /(?:\u001b\[|\u009b)[0-?]*[ -/]*[@-~]|(?:\u001b[\]PX^_]|[\u0090\u0098\u009d-\u009f])[^\u0007\u001b\u0090\u0098\u009c-\u009f]*(?:\u0007|\u001b\\|\u009c)|\u001b[ -/]*[0-~]/g;
|
|
101
|
+
/**
|
|
102
|
+
* What `sanitizeServerText` drops once the escape sequences are gone: the C0 controls but the
|
|
103
|
+
* whitespace ones (TAB to CR), DEL, the C1 controls, and every format character (General
|
|
104
|
+
* Category Cf: bidi controls, zero-width characters, U+00AD, U+FEFF, …).
|
|
105
|
+
*/
|
|
106
|
+
const DROPPED_CHAR = /[\u0000-\u0008\u000e-\u001f\u007f-\u009f\p{Cf}]/gu;
|
|
78
107
|
/**
|
|
79
108
|
* Make a string that originates in an attacker-controlled response — the error
|
|
80
109
|
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
81
110
|
* into an error message on stderr:
|
|
82
111
|
*
|
|
83
|
-
* -
|
|
112
|
+
* - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
|
|
113
|
+
* then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
|
|
84
114
|
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
85
115
|
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
86
116
|
* spoofing, title changes, OSC 52 clipboard writes).
|
|
87
|
-
* -
|
|
88
|
-
* reorder the visible message
|
|
117
|
+
* - Unicode format characters (General Category Cf) are dropped: the bidi controls
|
|
118
|
+
* (isBidiControl), so server text cannot reorder the visible message, and the
|
|
119
|
+
* invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
|
|
120
|
+
* two messages that look identical are identical.
|
|
89
121
|
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
90
122
|
* space and the ends are trimmed, so the text stays on one line and a server
|
|
91
123
|
* cannot forge an `Error:` line of its own.
|
|
124
|
+
* - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
|
|
125
|
+
* regex replace: a hostile error body costs neither seconds nor the heap. A caller that
|
|
126
|
+
* scrubs credentials does so first, so a cut never splits one.
|
|
92
127
|
*
|
|
93
128
|
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
94
129
|
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
95
|
-
* Written
|
|
130
|
+
* Written with escapes so no raw control byte appears in this source.
|
|
96
131
|
*/
|
|
97
132
|
export function sanitizeServerText(text) {
|
|
98
|
-
|
|
99
|
-
for (const ch of text) {
|
|
100
|
-
const n = ch.codePointAt(0) ?? 0;
|
|
101
|
-
const whitespaceControl = n >= 0x09 && n <= 0x0d;
|
|
102
|
-
if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
|
|
103
|
-
continue;
|
|
104
|
-
out += ch;
|
|
105
|
-
}
|
|
106
|
-
return out.replace(/\s+/g, " ").trim();
|
|
133
|
+
return cutText(text, MAX_SERVER_TEXT).replace(ESCAPE_SEQUENCE, "").replace(DROPPED_CHAR, "").replace(/\s+/g, " ").trim();
|
|
107
134
|
}
|
|
108
135
|
/**
|
|
109
136
|
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
@@ -200,14 +227,16 @@ function bodyBytes(value) {
|
|
|
200
227
|
*/
|
|
201
228
|
function plainHeaders(headers) {
|
|
202
229
|
const h = headers;
|
|
230
|
+
// Records built from an answer have no prototype: a header named `__proto__` is an own
|
|
231
|
+
// entry like any other, never the record's prototype (whose entries would read as headers).
|
|
203
232
|
if (typeof h.get === "function" && typeof h.forEach === "function") {
|
|
204
|
-
const record =
|
|
233
|
+
const record = Object.create(null);
|
|
205
234
|
h.forEach.call(headers, (value, name) => {
|
|
206
235
|
record[String(name).toLowerCase()] = value;
|
|
207
236
|
});
|
|
208
237
|
return record;
|
|
209
238
|
}
|
|
210
|
-
const record =
|
|
239
|
+
const record = Object.create(null);
|
|
211
240
|
for (const [name, value] of Object.entries(headers)) {
|
|
212
241
|
record[name.toLowerCase()] = value;
|
|
213
242
|
}
|
|
@@ -218,11 +247,29 @@ function headerValue(value) {
|
|
|
218
247
|
return Array.isArray(value) ? value[0] : value;
|
|
219
248
|
}
|
|
220
249
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
250
|
+
/** No declared big-integer members (`EngineOptions.bigIntegerKeys` left out). */
|
|
251
|
+
const NO_KEYS = new Set();
|
|
252
|
+
/**
|
|
253
|
+
* Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
|
|
254
|
+
* copied; anything else throws an CkanValidationError.
|
|
255
|
+
*/
|
|
256
|
+
export function bigIntegerKeysOption(value) {
|
|
257
|
+
if (value === undefined)
|
|
258
|
+
return NO_KEYS;
|
|
259
|
+
const iterable = typeof value === "object" && value !== null && typeof value[Symbol.iterator] === "function";
|
|
260
|
+
const names = iterable ? [...value] : [];
|
|
261
|
+
if (!iterable || names.some((key) => typeof key !== "string" || key === "")) {
|
|
262
|
+
throw new CkanValidationError("Invalid option bigIntegerKeys: expected a set of non-empty member names.");
|
|
263
|
+
}
|
|
264
|
+
return new Set(names);
|
|
265
|
+
}
|
|
221
266
|
export class RequestEngine {
|
|
222
267
|
// A real private field (not TypeScript's `private`): util.inspect, console.log and
|
|
223
268
|
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
224
269
|
// logged by accident. Messages use redactUrl.
|
|
225
270
|
#baseUrl;
|
|
271
|
+
/** `#baseUrl` without its userinfo: what every request URL starts with. */
|
|
272
|
+
#requestBase;
|
|
226
273
|
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
227
274
|
#credentials;
|
|
228
275
|
/**
|
|
@@ -240,14 +287,27 @@ export class RequestEngine {
|
|
|
240
287
|
maxResponseBytes;
|
|
241
288
|
sleep;
|
|
242
289
|
onRetry;
|
|
290
|
+
onWarning;
|
|
291
|
+
/** `EngineOptions.bigIntegerKeys`, copied. */
|
|
292
|
+
bigIntegerKeys;
|
|
243
293
|
constructor(options = {}) {
|
|
244
294
|
// The raw value, checked before the slash strip (the engine glues it into
|
|
245
295
|
// every URL), and here rather than only in the default transport: a library
|
|
246
296
|
// consumer that injects a custom transport would otherwise get no gating at
|
|
247
297
|
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
248
298
|
// the default.
|
|
249
|
-
|
|
250
|
-
|
|
299
|
+
const baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
300
|
+
// Requests go to the base URL as WHATWG parses it, the reading every check of it makes (the
|
|
301
|
+
// host, the cleartext warning, the Authorization header), never to a text-based reading of
|
|
302
|
+
// the string: a transport that reads the URL as text reaches the same host.
|
|
303
|
+
const parsed = new URL(baseUrl);
|
|
304
|
+
this.#baseUrl = parsed.href.replace(/\/+$/, "");
|
|
305
|
+
parsed.username = "";
|
|
306
|
+
parsed.password = "";
|
|
307
|
+
this.#requestBase = parsed.href.replace(/\/+$/, "");
|
|
308
|
+
// The userinfo as typed and as sent (WHATWG percent-encodes some of its characters).
|
|
309
|
+
const userinfos = [...new Set([...credentialsIn(baseUrl), ...credentialsIn(this.#baseUrl)])];
|
|
310
|
+
this.#credentials = userinfos.flatMap((raw) => {
|
|
251
311
|
try {
|
|
252
312
|
return [raw, decodeURIComponent(raw)];
|
|
253
313
|
}
|
|
@@ -255,8 +315,7 @@ export class RequestEngine {
|
|
|
255
315
|
return [raw];
|
|
256
316
|
}
|
|
257
317
|
});
|
|
258
|
-
this.#echoed =
|
|
259
|
-
.flatMap(echoedCredentialForms)
|
|
318
|
+
this.#echoed = [...new Set(userinfos.flatMap(echoedCredentialForms).flatMap(urlSpellings))]
|
|
260
319
|
.sort((a, b) => b.length - a.length);
|
|
261
320
|
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
262
321
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
@@ -273,6 +332,18 @@ export class RequestEngine {
|
|
|
273
332
|
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
274
333
|
this.onRetry =
|
|
275
334
|
options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
|
|
335
|
+
this.onWarning =
|
|
336
|
+
options.onWarning === undefined ? undefined : functionOption("onWarning", options.onWarning, () => { });
|
|
337
|
+
this.bigIntegerKeys = bigIntegerKeysOption(options.bigIntegerKeys);
|
|
338
|
+
}
|
|
339
|
+
/** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
|
|
340
|
+
warn(message) {
|
|
341
|
+
try {
|
|
342
|
+
this.onWarning?.(message);
|
|
343
|
+
}
|
|
344
|
+
catch {
|
|
345
|
+
// a logging hook is no reason to fail the request
|
|
346
|
+
}
|
|
276
347
|
}
|
|
277
348
|
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
278
349
|
async backOff(attempt, delayMs, url, status) {
|
|
@@ -300,6 +371,34 @@ export class RequestEngine {
|
|
|
300
371
|
scrub(text) {
|
|
301
372
|
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
302
373
|
}
|
|
374
|
+
/**
|
|
375
|
+
* An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
|
|
376
|
+
* with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
|
|
377
|
+
* credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
|
|
378
|
+
* a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
|
|
379
|
+
* with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
|
|
380
|
+
* decodes, and stays as sent; one too deeply nested to write again is left out.
|
|
381
|
+
*/
|
|
382
|
+
scrubBody(scrubbed) {
|
|
383
|
+
if (this.#echoed.length === 0 || !scrubbed.includes("\\"))
|
|
384
|
+
return scrubbed;
|
|
385
|
+
let decoded;
|
|
386
|
+
try {
|
|
387
|
+
decoded = JSON.parse(scrubbed);
|
|
388
|
+
}
|
|
389
|
+
catch {
|
|
390
|
+
return scrubbed;
|
|
391
|
+
}
|
|
392
|
+
let rewritten;
|
|
393
|
+
try {
|
|
394
|
+
rewritten = JSON.stringify(decoded);
|
|
395
|
+
}
|
|
396
|
+
catch {
|
|
397
|
+
return "";
|
|
398
|
+
}
|
|
399
|
+
const spelled = this.#echoed.map((form) => JSON.stringify(form).slice(1, -1));
|
|
400
|
+
return spelled.some((form) => rewritten.includes(form)) ? redactSecrets(rewritten, spelled) : scrubbed;
|
|
401
|
+
}
|
|
303
402
|
/**
|
|
304
403
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
305
404
|
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
@@ -337,7 +436,7 @@ export class RequestEngine {
|
|
|
337
436
|
composeUrl(path, query, withUserinfo) {
|
|
338
437
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
339
438
|
const qs = query ? buildQueryString(query) : "";
|
|
340
|
-
const base = withUserinfo ? this.#baseUrl :
|
|
439
|
+
const base = withUserinfo ? this.#baseUrl : this.#requestBase;
|
|
341
440
|
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
342
441
|
}
|
|
343
442
|
/**
|
|
@@ -367,6 +466,24 @@ export class RequestEngine {
|
|
|
367
466
|
clearTimeout(timer);
|
|
368
467
|
}
|
|
369
468
|
}
|
|
469
|
+
/**
|
|
470
|
+
* The URL a redirect leads to as messages and errors show it: the Location with the base URL's
|
|
471
|
+
* credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
|
|
472
|
+
* spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
|
|
473
|
+
* left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
|
|
474
|
+
*/
|
|
475
|
+
shownTarget(location, shown) {
|
|
476
|
+
const sent = this.scrub(location);
|
|
477
|
+
try {
|
|
478
|
+
const target = new URL(sent, shown);
|
|
479
|
+
target.username = "";
|
|
480
|
+
target.password = "";
|
|
481
|
+
return target.href;
|
|
482
|
+
}
|
|
483
|
+
catch {
|
|
484
|
+
return sent;
|
|
485
|
+
}
|
|
486
|
+
}
|
|
370
487
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
371
488
|
async request(method, path, options = { accept: "application/json" }) {
|
|
372
489
|
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
@@ -374,6 +491,9 @@ export class RequestEngine {
|
|
|
374
491
|
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
375
492
|
// such as fetch also refuses a URL with credentials outright.
|
|
376
493
|
let url = this.composeUrl(path, options.query, false);
|
|
494
|
+
// The URL as messages, errors and `RawResponse.url` show it: after a redirect, the Location
|
|
495
|
+
// with the base URL's credentials scrubbed (shownTarget); `url` is what is requested.
|
|
496
|
+
let shown = url;
|
|
377
497
|
let headers = {
|
|
378
498
|
Accept: options.accept,
|
|
379
499
|
"User-Agent": this.userAgent,
|
|
@@ -407,7 +527,7 @@ export class RequestEngine {
|
|
|
407
527
|
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
408
528
|
throw cause;
|
|
409
529
|
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
410
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
530
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
411
531
|
cause: this.scrubCause(cause),
|
|
412
532
|
});
|
|
413
533
|
}
|
|
@@ -415,14 +535,14 @@ export class RequestEngine {
|
|
|
415
535
|
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
416
536
|
const invalid = responseProblem(response);
|
|
417
537
|
if (invalid !== undefined) {
|
|
418
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
538
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport returned an invalid response (${invalid}).`);
|
|
419
539
|
}
|
|
420
540
|
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
421
541
|
// did (fetch's default) may have carried the Authorization header to another
|
|
422
542
|
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
423
543
|
const finalUrl = response.url;
|
|
424
544
|
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
425
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
545
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: the transport followed a redirect to another origin ` +
|
|
426
546
|
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
427
547
|
`(HttpRequest.redirect is "manual").`);
|
|
428
548
|
}
|
|
@@ -433,7 +553,7 @@ export class RequestEngine {
|
|
|
433
553
|
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
434
554
|
// custom one may have read everything.
|
|
435
555
|
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
436
|
-
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(
|
|
556
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(shown))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
437
557
|
}
|
|
438
558
|
const retryable = status === 429 || status === 503;
|
|
439
559
|
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
@@ -446,7 +566,7 @@ export class RequestEngine {
|
|
|
446
566
|
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
447
567
|
// zero-delay burst against a server that had just asked for less load.
|
|
448
568
|
const backoff = this.retryDelayMs * attempt;
|
|
449
|
-
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff),
|
|
569
|
+
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), shown, status);
|
|
450
570
|
continue;
|
|
451
571
|
}
|
|
452
572
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
@@ -462,7 +582,7 @@ export class RequestEngine {
|
|
|
462
582
|
// never reaches a transport (a custom one may not check the scheme).
|
|
463
583
|
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
464
584
|
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
465
|
-
`for ${method} ${cutForMessage(redactUrl(
|
|
585
|
+
`for ${method} ${cutForMessage(redactUrl(shown))}`);
|
|
466
586
|
}
|
|
467
587
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
468
588
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
@@ -483,6 +603,7 @@ export class RequestEngine {
|
|
|
483
603
|
headers = engineHeadersOnly(headers);
|
|
484
604
|
}
|
|
485
605
|
url = nextUrl.toString();
|
|
606
|
+
shown = this.shownTarget(location, shown);
|
|
486
607
|
redirects += 1;
|
|
487
608
|
continue;
|
|
488
609
|
}
|
|
@@ -497,47 +618,72 @@ export class RequestEngine {
|
|
|
497
618
|
const detail = target !== undefined
|
|
498
619
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
499
620
|
: location
|
|
500
|
-
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
621
|
+
? `redirect to ${cutForMessage(sanitizeServerText(this.scrub(location)))} not followed`
|
|
501
622
|
: "redirect not followed (no Location header)";
|
|
502
|
-
throw new CkanApiError({ status, url, method, body: this.scrub(body
|
|
623
|
+
throw new CkanApiError({ status, url: shown, method, body: this.scrubBody(this.scrub(errorBodyText(body))), detail });
|
|
503
624
|
}
|
|
504
|
-
throw this.toApiError(method,
|
|
625
|
+
throw this.toApiError(method, shown, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
505
626
|
retries: attempt,
|
|
506
627
|
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
507
628
|
});
|
|
508
629
|
}
|
|
509
|
-
return { data: body, contentType, status, url };
|
|
630
|
+
return { data: body, contentType, status, url: shown };
|
|
510
631
|
}
|
|
511
632
|
}
|
|
512
|
-
/** Perform a GET expecting JSON and parse it into `T
|
|
633
|
+
/** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
|
|
513
634
|
async getJson(path, query) {
|
|
635
|
+
return (await this.getAnswer(path, query)).value;
|
|
636
|
+
}
|
|
637
|
+
/**
|
|
638
|
+
* Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
|
|
639
|
+
* userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
|
|
640
|
+
* MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
|
|
641
|
+
* (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
|
|
642
|
+
* one failed; the client prefixes its own envelope and shape errors with it too.
|
|
643
|
+
*
|
|
644
|
+
* The text is never changed silently: bytes that are not valid in the charset, and lone
|
|
645
|
+
* surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
|
|
646
|
+
* well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
|
|
647
|
+
* `onWarning` is told how many. One bad character never sinks a whole answer.
|
|
648
|
+
*/
|
|
649
|
+
async getAnswer(path, query) {
|
|
514
650
|
const res = await this.request("GET", path, { query, accept: "application/json" });
|
|
515
|
-
const
|
|
651
|
+
const url = redactUrl(res.url);
|
|
652
|
+
const request = `GET ${cutForMessage(url)}`;
|
|
653
|
+
const decoded = decodeBody(res.data, res.contentType, request);
|
|
654
|
+
if (decoded.text.trim() === "") {
|
|
655
|
+
// A broken answer, not "no results".
|
|
656
|
+
throw new CkanParseError(`${request}: empty response body.`);
|
|
657
|
+
}
|
|
658
|
+
const repaired = wellFormedEscapes(decoded.text);
|
|
659
|
+
// Declared big integers are quoted first, so JSON.parse delivers them exactly.
|
|
660
|
+
const { json, inexact } = scanNumbers(repaired.json, this.bigIntegerKeys);
|
|
516
661
|
let parsed;
|
|
517
662
|
try {
|
|
518
|
-
parsed = JSON.parse(
|
|
663
|
+
parsed = JSON.parse(json);
|
|
519
664
|
}
|
|
520
665
|
catch (cause) {
|
|
521
|
-
//
|
|
522
|
-
//
|
|
523
|
-
//
|
|
524
|
-
|
|
525
|
-
const
|
|
526
|
-
|
|
527
|
-
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
528
|
-
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
529
|
-
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
666
|
+
// A site that is not a CKAN (or not at this path) typically answers with an HTML page and
|
|
667
|
+
// HTTP 200: name its type. The Content-Type is server text (latin-1 decoded, so a 0x9B byte
|
|
668
|
+
// is the 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
669
|
+
const type = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
|
|
670
|
+
const hint = type !== "" && !/json/i.test(type) ? ` (expected JSON, got Content-Type "${type}")` : "";
|
|
671
|
+
throw new CkanParseError(`${request}: the answer is not valid JSON${hint}.`, { cause: this.scrubCause(cause) });
|
|
530
672
|
}
|
|
531
673
|
// JSON.parse rounds and overflows numbers silently: refuse the answer rather than print a
|
|
532
674
|
// changed value.
|
|
533
|
-
const inexact = firstInexactNumber(text);
|
|
534
675
|
if (inexact !== undefined) {
|
|
535
|
-
throw new CkanParseError(
|
|
676
|
+
throw new CkanParseError(`${request}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
|
|
536
677
|
}
|
|
537
|
-
|
|
678
|
+
const replaced = decoded.replaced + repaired.replaced;
|
|
679
|
+
if (replaced > 0) {
|
|
680
|
+
this.warn(`${request}: ${replaced} ${replaced === 1 ? "character" : "characters"} of the answer ${replaced === 1 ? "was" : "were"} ` +
|
|
681
|
+
"not valid text (invalid bytes or a lone surrogate) and replaced with U+FFFD.");
|
|
682
|
+
}
|
|
683
|
+
return { value: parsed, url, request };
|
|
538
684
|
}
|
|
539
685
|
toApiError(method, url, status, body, hint, retry) {
|
|
540
|
-
const text = this.scrub(body
|
|
686
|
+
const text = this.scrub(errorBodyText(body));
|
|
541
687
|
let detail;
|
|
542
688
|
try {
|
|
543
689
|
const parsed = JSON.parse(text);
|
|
@@ -564,28 +710,24 @@ export class RequestEngine {
|
|
|
564
710
|
catch {
|
|
565
711
|
// Non-JSON error body; leave detail undefined.
|
|
566
712
|
}
|
|
567
|
-
// `detail` came from the response body
|
|
713
|
+
// `detail` came from the response body, decoded: scrubbed again (a JSON escape hid the
|
|
714
|
+
// credential from the scrub above), then stripped of control characters so a hostile
|
|
568
715
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
569
716
|
if (detail !== undefined)
|
|
570
|
-
detail = sanitizeServerText(detail);
|
|
717
|
+
detail = sanitizeServerText(this.scrub(detail));
|
|
571
718
|
if (hint !== undefined)
|
|
572
719
|
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
573
720
|
return new CkanApiError({
|
|
574
721
|
status,
|
|
575
722
|
url,
|
|
576
723
|
method,
|
|
577
|
-
body: text,
|
|
724
|
+
body: this.scrubBody(text),
|
|
578
725
|
detail,
|
|
579
726
|
retries: retry.retries,
|
|
580
727
|
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
581
728
|
});
|
|
582
729
|
}
|
|
583
730
|
}
|
|
584
|
-
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
585
|
-
function withoutUserinfo(url) {
|
|
586
|
-
const [userinfo] = credentialsIn(url);
|
|
587
|
-
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
588
|
-
}
|
|
589
731
|
/**
|
|
590
732
|
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
591
733
|
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
@@ -611,18 +753,143 @@ function originOf(url) {
|
|
|
611
753
|
* Decode a response body by the charset of its Content-Type (UTF-8 when none is
|
|
612
754
|
* given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
|
|
613
755
|
* that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
|
|
614
|
-
* sends UTF-8; this matters for proxies and mirrors that re-encode.
|
|
756
|
+
* sends UTF-8; this matters for proxies and mirrors that re-encode. An unknown charset
|
|
757
|
+
* label is a CkanParseError naming `request`. Bytes that are not valid in the charset become
|
|
758
|
+
* U+FFFD, and `replaced` counts them: exactly for UTF-8 and the single-byte charsets; for
|
|
759
|
+
* UTF-16 a U+FFFD the server sent counts too.
|
|
615
760
|
*/
|
|
616
|
-
function decodeBody(body, contentType,
|
|
761
|
+
function decodeBody(body, contentType, request) {
|
|
617
762
|
const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
|
|
618
763
|
let decoder;
|
|
619
764
|
try {
|
|
620
|
-
decoder = new TextDecoder(charset);
|
|
765
|
+
decoder = new TextDecoder(charset, { fatal: true });
|
|
766
|
+
}
|
|
767
|
+
catch {
|
|
768
|
+
throw new CkanParseError(`${request}: unsupported response charset "${cutForMessage(sanitizeServerText(charset))}".`);
|
|
769
|
+
}
|
|
770
|
+
try {
|
|
771
|
+
return { text: decoder.decode(body), replaced: 0 };
|
|
772
|
+
}
|
|
773
|
+
catch (err) {
|
|
774
|
+
if (isStringTooLong(err))
|
|
775
|
+
throw tooLargeForText(request, body);
|
|
776
|
+
// Not valid in the charset: decode with replacement, and count what was replaced.
|
|
777
|
+
let text;
|
|
778
|
+
try {
|
|
779
|
+
text = new TextDecoder(decoder.encoding).decode(body);
|
|
780
|
+
}
|
|
781
|
+
catch (again) {
|
|
782
|
+
if (isStringTooLong(again))
|
|
783
|
+
throw tooLargeForText(request, body);
|
|
784
|
+
throw again;
|
|
785
|
+
}
|
|
786
|
+
const sent = decoder.encoding === "utf-8" ? countBytes(body, REPLACEMENT_UTF8) : 0;
|
|
787
|
+
return { text, replaced: countChar(text, 0xfffd) - sent };
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* True for V8's error for a string past its longest one (2^29 − 24 characters on 64-bit): a body
|
|
792
|
+
* of 512 MB and more, possible once `maxResponseBytes` is lifted, can't be decoded into one.
|
|
793
|
+
*/
|
|
794
|
+
function isStringTooLong(err) {
|
|
795
|
+
return err?.code === "ERR_STRING_TOO_LONG";
|
|
796
|
+
}
|
|
797
|
+
/** The parse error for a body too large to decode: it used to escape as a plain Error ("Unexpected error"). */
|
|
798
|
+
function tooLargeForText(request, body) {
|
|
799
|
+
return new CkanParseError(`${request}: the answer is too large to read as text (${body.byteLength} bytes; the longest string is ${MAX_STRING_LENGTH} characters).`);
|
|
800
|
+
}
|
|
801
|
+
/**
|
|
802
|
+
* The body of an error answer as text, kept on the error (`body`). One too large to be a string
|
|
803
|
+
* (maxResponseBytes lifted) is left out: the status is the error.
|
|
804
|
+
*/
|
|
805
|
+
function errorBodyText(body) {
|
|
806
|
+
try {
|
|
807
|
+
return body.toString("utf8");
|
|
808
|
+
}
|
|
809
|
+
catch (err) {
|
|
810
|
+
if (!isStringTooLong(err))
|
|
811
|
+
throw err;
|
|
812
|
+
return "";
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
/** U+FFFD in UTF-8. */
|
|
816
|
+
const REPLACEMENT_UTF8 = Buffer.from([0xef, 0xbf, 0xbd]);
|
|
817
|
+
/** How often `needle` occurs in `body`. */
|
|
818
|
+
function countBytes(body, needle) {
|
|
819
|
+
let n = 0;
|
|
820
|
+
for (let at = body.indexOf(needle); at !== -1; at = body.indexOf(needle, at + needle.length))
|
|
821
|
+
n++;
|
|
822
|
+
return n;
|
|
823
|
+
}
|
|
824
|
+
/** How often the UTF-16 unit `code` occurs in `text`. */
|
|
825
|
+
function countChar(text, code) {
|
|
826
|
+
let n = 0;
|
|
827
|
+
for (let i = 0; i < text.length; i++)
|
|
828
|
+
if (text.charCodeAt(i) === code)
|
|
829
|
+
n++;
|
|
830
|
+
return n;
|
|
831
|
+
}
|
|
832
|
+
/** A JSON escape of a surrogate, `\uD800`–`\uDFFF`, anywhere in a text (a cheap pre-check). */
|
|
833
|
+
const SURROGATE_ESCAPE = /\\u[dD][89a-fA-F][0-9a-fA-F]{2}/;
|
|
834
|
+
/** The value of the four hex digits of a `\uXXXX` escape whose `u` is at `at`, or -1. */
|
|
835
|
+
function escapeAt(json, at) {
|
|
836
|
+
if (json.charCodeAt(at) !== 0x75)
|
|
837
|
+
return -1;
|
|
838
|
+
const hex = json.slice(at + 1, at + 5);
|
|
839
|
+
return /^[0-9a-fA-F]{4}$/.test(hex) ? parseInt(hex, 16) : -1;
|
|
840
|
+
}
|
|
841
|
+
/**
|
|
842
|
+
* `json` (decoded answer text) with every lone surrogate written as a JSON escape (`\ud800` not
|
|
843
|
+
* followed by a low-surrogate escape, or a `\udc00` not preceded by a high one) rewritten as
|
|
844
|
+
* `\ufffd`, and how many. JSON.parse would make such an escape half a character, which a JSON
|
|
845
|
+
* printer writes back as `\ud800` and `jq` refuses (the whole output with it). A raw lone
|
|
846
|
+
* surrogate cannot occur: TextDecoder never makes one. One linear pass, and only when the text
|
|
847
|
+
* holds a surrogate escape at all; the text is copied only when one is lone.
|
|
848
|
+
*/
|
|
849
|
+
function wellFormedEscapes(json) {
|
|
850
|
+
if (!SURROGATE_ESCAPE.test(json))
|
|
851
|
+
return { json, replaced: 0 };
|
|
852
|
+
const out = new TextBuilder();
|
|
853
|
+
let copied = 0;
|
|
854
|
+
let replaced = 0;
|
|
855
|
+
for (let i = 0; i < json.length; i++) {
|
|
856
|
+
if (json.charCodeAt(i) !== 0x5c)
|
|
857
|
+
continue;
|
|
858
|
+
const code = escapeAt(json, i + 1);
|
|
859
|
+
if (code < 0xd800 || code > 0xdfff) {
|
|
860
|
+
i++; // an escape of another kind (\", \\, \n, \u0041): skip its second character
|
|
861
|
+
continue;
|
|
862
|
+
}
|
|
863
|
+
const pairedLow = code <= 0xdbff && json.charCodeAt(i + 6) === 0x5c && isLowSurrogate(escapeAt(json, i + 7));
|
|
864
|
+
if (pairedLow) {
|
|
865
|
+
i += 11;
|
|
866
|
+
continue;
|
|
867
|
+
}
|
|
868
|
+
out.push(json.slice(copied, i), "\\ufffd");
|
|
869
|
+
copied = i + 6;
|
|
870
|
+
replaced++;
|
|
871
|
+
i += 5;
|
|
872
|
+
}
|
|
873
|
+
if (replaced === 0)
|
|
874
|
+
return { json, replaced };
|
|
875
|
+
out.push(json.slice(copied));
|
|
876
|
+
return { json: out.toString(), replaced };
|
|
877
|
+
}
|
|
878
|
+
function isLowSurrogate(code) {
|
|
879
|
+
return code >= 0xdc00 && code <= 0xdfff;
|
|
880
|
+
}
|
|
881
|
+
/**
|
|
882
|
+
* `form` and its spelling in a URL (`encodeURIComponent`): a server that puts a credential into
|
|
883
|
+
* a URL, a redirect's Location, percent-encodes it. A form that has no such spelling (a lone
|
|
884
|
+
* surrogate) is kept as it is.
|
|
885
|
+
*/
|
|
886
|
+
function urlSpellings(form) {
|
|
887
|
+
try {
|
|
888
|
+
return [form, encodeURIComponent(form)];
|
|
621
889
|
}
|
|
622
890
|
catch {
|
|
623
|
-
|
|
891
|
+
return [form];
|
|
624
892
|
}
|
|
625
|
-
return decoder.decode(body);
|
|
626
893
|
}
|
|
627
894
|
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
628
895
|
function resolveLocation(location, base) {
|
|
@@ -637,16 +904,20 @@ function resolveLocation(location, base) {
|
|
|
637
904
|
}
|
|
638
905
|
/** A JSON number token, matched at a given position (sticky). */
|
|
639
906
|
const JSON_NUMBER = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
907
|
+
/** The smallest positive normal double, 2^-1022. Below it a double has fewer significant digits. */
|
|
908
|
+
const MIN_NORMAL = 2.2250738585072014e-308;
|
|
640
909
|
/**
|
|
641
910
|
* Whether the double `value` (finite) prints as the number `literal` wrote, checked where a
|
|
642
|
-
* double can be off
|
|
643
|
-
*
|
|
644
|
-
*
|
|
645
|
-
* `
|
|
646
|
-
*
|
|
911
|
+
* double can be off in its leading digits: past 2^53, where every double is an integer, at 0,
|
|
912
|
+
* which a literal too small for a double becomes, and below the smallest normal double, where a
|
|
913
|
+
* subnormal has fewer significant digits (`3e-324` becomes `5e-324`). There the CLI prints
|
|
914
|
+
* `String(value)`, the shortest decimal that reads back as the same double: `12345678901234567890`
|
|
915
|
+
* prints as `12345678901234567000`, a changed value, while `1E23`, `1.7976931348623157E308` or
|
|
916
|
+
* `5e-324` (a double a server printed in its own shortest form) print as the same number.
|
|
647
917
|
*/
|
|
648
918
|
function exactAtTheEdges(literal, value) {
|
|
649
|
-
|
|
919
|
+
const size = Math.abs(value);
|
|
920
|
+
if (value !== 0 && size <= Number.MAX_SAFE_INTEGER && size >= MIN_NORMAL)
|
|
650
921
|
return true;
|
|
651
922
|
const written = decimalOf(literal);
|
|
652
923
|
if (value === 0)
|
|
@@ -657,30 +928,87 @@ function exactAtTheEdges(literal, value) {
|
|
|
657
928
|
/** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
|
|
658
929
|
function decimalOf(text) {
|
|
659
930
|
const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
931
|
+
// The zeros are counted with index loops: `/0+$/` tried a match at every zero of a run that
|
|
932
|
+
// doesn't end the text, quadratic in the run, and one `1.<64 000 zeros>1e300` in an answer held
|
|
933
|
+
// the CLI for seconds, past --timeout (the scan runs once the answer is in).
|
|
934
|
+
const all = whole + fraction;
|
|
935
|
+
let start = 0;
|
|
936
|
+
while (start < all.length && all.charCodeAt(start) === 0x30)
|
|
937
|
+
start++;
|
|
938
|
+
let end = all.length;
|
|
939
|
+
while (end > start && all.charCodeAt(end - 1) === 0x30)
|
|
940
|
+
end--;
|
|
941
|
+
return { negative: sign === "-", digits: all.slice(start, end), exponent: Number(exp) - fraction.length + (all.length - end) };
|
|
942
|
+
}
|
|
943
|
+
/** A JSON number token as the grammar allows it, matched at a given position (sticky). */
|
|
944
|
+
const STRICT_JSON_NUMBER = /-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
945
|
+
/** Whether the member name `json` holds between `start` and `end` (its quotes) is one of `keys`, compared in place. */
|
|
946
|
+
function isDeclared(json, start, end, keys) {
|
|
947
|
+
for (const key of keys) {
|
|
948
|
+
if (end - start - 1 === key.length && json.startsWith(key, start + 1))
|
|
949
|
+
return true;
|
|
950
|
+
}
|
|
951
|
+
return false;
|
|
952
|
+
}
|
|
953
|
+
/** The index of the first character at or after `i` that is not JSON whitespace. */
|
|
954
|
+
function skipSpace(json, i) {
|
|
955
|
+
for (; i < json.length; i++) {
|
|
956
|
+
const c = json.charCodeAt(i);
|
|
957
|
+
if (c !== 0x20 && c !== 0x09 && c !== 0x0a && c !== 0x0d)
|
|
958
|
+
break;
|
|
959
|
+
}
|
|
960
|
+
return i;
|
|
663
961
|
}
|
|
664
962
|
/**
|
|
665
|
-
*
|
|
666
|
-
*
|
|
667
|
-
*
|
|
668
|
-
*
|
|
669
|
-
*
|
|
670
|
-
*
|
|
963
|
+
* One linear pass over `json` (the text of an answer) that skips strings and looks at every
|
|
964
|
+
* number literal:
|
|
965
|
+
* - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
|
|
966
|
+
* 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
|
|
967
|
+
* it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
|
|
968
|
+
* so a malformed one (`01`) still fails JSON.parse;
|
|
969
|
+
* - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
|
|
970
|
+
* cannot carry exactly is reported in `inexact`.
|
|
971
|
+
* Nothing is allocated per value: a member name is compared in place (only a name written with
|
|
972
|
+
* an escape is decoded), and the text is copied only when a declared member is found. No
|
|
973
|
+
* reviver, which would recurse per nesting level and overflow on deep data.
|
|
671
974
|
*/
|
|
672
|
-
function
|
|
975
|
+
export function scanNumbers(json, bigIntegerKeys = NO_KEYS) {
|
|
976
|
+
const keys = bigIntegerKeys.size === 0 ? undefined : [...bigIntegerKeys];
|
|
977
|
+
let inexact;
|
|
978
|
+
let out;
|
|
979
|
+
let copied = 0;
|
|
673
980
|
for (let i = 0; i < json.length; i++) {
|
|
674
981
|
const c = json.charCodeAt(i);
|
|
675
982
|
if (c === 0x22) {
|
|
676
983
|
// a string: skip to its closing quote, past escapes
|
|
984
|
+
const start = i;
|
|
985
|
+
let escaped = false;
|
|
677
986
|
for (i++; i < json.length; i++) {
|
|
678
987
|
const d = json.charCodeAt(i);
|
|
679
|
-
if (d === 0x5c)
|
|
988
|
+
if (d === 0x5c) {
|
|
989
|
+
escaped = true;
|
|
680
990
|
i++;
|
|
991
|
+
}
|
|
681
992
|
else if (d === 0x22)
|
|
682
993
|
break;
|
|
683
994
|
}
|
|
995
|
+
if (keys === undefined)
|
|
996
|
+
continue;
|
|
997
|
+
// A member name is a string followed by a colon; is it a declared one with a number value?
|
|
998
|
+
const colon = skipSpace(json, i + 1);
|
|
999
|
+
if (json.charCodeAt(colon) !== 0x3a)
|
|
1000
|
+
continue;
|
|
1001
|
+
if (!(escaped ? bigIntegerKeys.has(decodedName(json.slice(start, i + 1))) : isDeclared(json, start, i, keys)))
|
|
1002
|
+
continue;
|
|
1003
|
+
const at = skipSpace(json, colon + 1);
|
|
1004
|
+
STRICT_JSON_NUMBER.lastIndex = at;
|
|
1005
|
+
const literal = STRICT_JSON_NUMBER.exec(json)?.[0];
|
|
1006
|
+
if (literal === undefined)
|
|
1007
|
+
continue;
|
|
1008
|
+
out ??= new TextBuilder();
|
|
1009
|
+
out.push(json.slice(copied, at), '"', literal, '"');
|
|
1010
|
+
copied = at + literal.length;
|
|
1011
|
+
i = copied - 1;
|
|
684
1012
|
}
|
|
685
1013
|
else if (c === 0x2d || (c >= 0x30 && c <= 0x39)) {
|
|
686
1014
|
JSON_NUMBER.lastIndex = i;
|
|
@@ -688,10 +1016,37 @@ function firstInexactNumber(json) {
|
|
|
688
1016
|
if (literal === undefined)
|
|
689
1017
|
continue;
|
|
690
1018
|
const value = Number(literal);
|
|
691
|
-
if (!Number.isFinite(value) || !exactAtTheEdges(literal, value))
|
|
692
|
-
|
|
1019
|
+
if (inexact === undefined && (!Number.isFinite(value) || !exactAtTheEdges(literal, value))) {
|
|
1020
|
+
inexact = literal;
|
|
1021
|
+
if (keys === undefined)
|
|
1022
|
+
break;
|
|
1023
|
+
}
|
|
693
1024
|
i += literal.length - 1;
|
|
694
1025
|
}
|
|
695
1026
|
}
|
|
696
|
-
|
|
1027
|
+
if (out === undefined)
|
|
1028
|
+
return { json, inexact };
|
|
1029
|
+
out.push(json.slice(copied));
|
|
1030
|
+
return { json: out.toString(), inexact };
|
|
1031
|
+
}
|
|
1032
|
+
/** A member name written with an escape, decoded; a name JSON.parse can't read matches nothing. */
|
|
1033
|
+
function decodedName(quoted) {
|
|
1034
|
+
try {
|
|
1035
|
+
return String(JSON.parse(quoted));
|
|
1036
|
+
}
|
|
1037
|
+
catch {
|
|
1038
|
+
return "";
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
/**
|
|
1042
|
+
* The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
|
|
1043
|
+
* undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
|
|
1044
|
+
* 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
|
|
1045
|
+
* `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
|
|
1046
|
+
* (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
|
|
1047
|
+
* `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
|
|
1048
|
+
* precision, as JSON means it. {@link scanNumbers} without declared members.
|
|
1049
|
+
*/
|
|
1050
|
+
export function firstInexactNumber(json) {
|
|
1051
|
+
return scanNumbers(json).inexact;
|
|
697
1052
|
}
|