@maschinenlesbar.org/ckan-cli 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +25 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +64 -23
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +26 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +152 -45
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +52 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +155 -17
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.d.ts +441 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.js +1211 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +2 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +8 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +21 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.d.ts +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.js +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.d.ts +80 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.js +124 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/package.json +25 -0
- package/dist/src/cli/index.js +3 -1
- package/dist/src/cli/io.d.ts +17 -0
- package/dist/src/cli/io.js +14 -0
- package/dist/src/cli/limits.d.ts +19 -0
- package/dist/src/cli/limits.js +53 -0
- package/dist/src/cli/program.js +3 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +17 -16
- package/dist/src/cli/shared.d.ts +4 -0
- package/dist/src/cli/shared.js +16 -1
- package/node_modules/commander/Readme.md +6 -32
- package/node_modules/commander/index.js +1 -1
- package/node_modules/commander/lib/command.js +22 -9
- package/node_modules/commander/lib/help.js +6 -2
- package/node_modules/commander/lib/option.js +1 -1
- package/node_modules/commander/package.json +2 -1
- package/node_modules/commander/typings/index.d.ts +6 -3
- package/package.json +2 -2
|
@@ -27,14 +27,17 @@ export function redactUrl(url) {
|
|
|
27
27
|
/**
|
|
28
28
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
29
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
31
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
31
32
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
33
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
34
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
34
35
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
35
36
|
*/
|
|
36
37
|
export function credentialsIn(value) {
|
|
37
|
-
|
|
38
|
+
// WHATWG skips any number of "/" and "\" after a special scheme (`http:///u:pw@h`,
|
|
39
|
+
// `https:\\u:pw@h` and `http:u:pw@h` all send `u:pw`), so its userinfo starts after them.
|
|
40
|
+
const scheme = SPECIAL_SCHEME_START.exec(value) ?? /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
41
|
if (scheme === null)
|
|
39
42
|
return [];
|
|
40
43
|
const rest = value.slice(scheme[0].length);
|
|
@@ -47,23 +50,138 @@ export function credentialsIn(value) {
|
|
|
47
50
|
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
48
51
|
}
|
|
49
52
|
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
50
|
-
// first "/", "?" or "#"
|
|
51
|
-
|
|
53
|
+
// first "/", "?" or "#", and for a special scheme such as http(s) a "\", where WHATWG ends
|
|
54
|
+
// it too: `https://u:pw@a.example\@b.example` sends `u:pw` to a.example); in one that doesn't,
|
|
55
|
+
// at the last "@" of the value.
|
|
56
|
+
const stop = SPECIAL_SCHEME.test(scheme[0]) ? /[/?#\\]|$/ : /[/?#]|$/;
|
|
57
|
+
const authority = parses ? rest.slice(0, rest.search(stop)) : rest;
|
|
52
58
|
const end = authority.lastIndexOf("@");
|
|
53
59
|
return end > 0 ? [rest.slice(0, end)] : [];
|
|
54
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
63
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
64
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
65
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
66
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
67
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
68
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
69
|
+
* other text a bare `a/b@c` is no credential.
|
|
70
|
+
*/
|
|
71
|
+
export function typedUrlCredentials(value) {
|
|
72
|
+
if (value === "" || credentialsIn(value).length > 0 || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
|
|
73
|
+
return credentialsIn(value);
|
|
74
|
+
const prefixed = credentialsIn(`http://${value}`);
|
|
75
|
+
if (prefixed.length > 0)
|
|
76
|
+
return prefixed;
|
|
77
|
+
const slashes = /[/\\]+/.exec(value);
|
|
78
|
+
const at = value.lastIndexOf("@");
|
|
79
|
+
if (slashes === null || slashes.index + slashes[0].length >= at)
|
|
80
|
+
return [];
|
|
81
|
+
return [value.slice(slashes.index + slashes[0].length, at)];
|
|
82
|
+
}
|
|
83
|
+
/** The schemes WHATWG parses as special (`ftp`, `file`, `http(s)`, `ws(s)`): their authority also ends at a backslash. */
|
|
84
|
+
const SPECIAL_SCHEME = /^(?:ftp|file|https?|wss?):[/\\]*$/i;
|
|
85
|
+
/** A special scheme at the start of a value, with the slashes and backslashes WHATWG skips after it. */
|
|
86
|
+
const SPECIAL_SCHEME_START = /^(?:ftp|file|https?|wss?):[/\\]*/i;
|
|
87
|
+
/** A scheme character (RFC 3986: letters, digits, `+`, `.`, `-`). */
|
|
88
|
+
function isSchemeChar(code) {
|
|
89
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39) || code === 0x2b || code === 0x2e || code === 0x2d;
|
|
90
|
+
}
|
|
91
|
+
/** An ASCII letter, with which a scheme starts. */
|
|
92
|
+
function isLetter(code) {
|
|
93
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Where the scheme of the `://` at `at` starts: the first letter of the run of scheme characters
|
|
97
|
+
* before it, looking back no further than `from`; -1 when there is none (`1://`, ` ://`). A run
|
|
98
|
+
* led by other scheme characters (`-https://`) starts at its first letter.
|
|
99
|
+
*/
|
|
100
|
+
function schemeStart(text, at, from) {
|
|
101
|
+
let start = at;
|
|
102
|
+
while (start > from && isSchemeChar(text.charCodeAt(start - 1)))
|
|
103
|
+
start--;
|
|
104
|
+
while (start < at && !isLetter(text.charCodeAt(start)))
|
|
105
|
+
start++;
|
|
106
|
+
return start < at ? start : -1;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
110
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
111
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
112
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
113
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
114
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
115
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
116
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
117
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
118
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
119
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
120
|
+
*/
|
|
121
|
+
export function redactUrlUserinfo(text) {
|
|
122
|
+
const parts = new TextBuilder();
|
|
123
|
+
let copied = 0;
|
|
124
|
+
let from = 0;
|
|
125
|
+
for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", from)) {
|
|
126
|
+
const begin = at + 3;
|
|
127
|
+
let end = begin;
|
|
128
|
+
let lastAt = -1;
|
|
129
|
+
for (; end < text.length; end++) {
|
|
130
|
+
const c = text.charCodeAt(end);
|
|
131
|
+
if (c === 0x2f || c === 0x3f || c === 0x23 || c === 0x27 || /\s/.test(text[end]))
|
|
132
|
+
break;
|
|
133
|
+
if (c === 0x40)
|
|
134
|
+
lastAt = end;
|
|
135
|
+
}
|
|
136
|
+
if (lastAt !== -1 && schemeStart(text, at, from) !== -1) {
|
|
137
|
+
parts.push(text.slice(copied, begin), "***");
|
|
138
|
+
copied = lastAt;
|
|
139
|
+
}
|
|
140
|
+
from = begin;
|
|
141
|
+
}
|
|
142
|
+
if (copied === 0)
|
|
143
|
+
return text;
|
|
144
|
+
parts.push(text.slice(copied));
|
|
145
|
+
return parts.toString();
|
|
146
|
+
}
|
|
147
|
+
/** How many pieces a TextBuilder joins at a time. */
|
|
148
|
+
const BUILDER_BATCH = 16_384;
|
|
149
|
+
/**
|
|
150
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
151
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
152
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
153
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
154
|
+
* either way.
|
|
155
|
+
*/
|
|
156
|
+
export class TextBuilder {
|
|
157
|
+
#done = [];
|
|
158
|
+
#batch = [];
|
|
159
|
+
push(...pieces) {
|
|
160
|
+
for (const piece of pieces)
|
|
161
|
+
this.#batch.push(piece);
|
|
162
|
+
if (this.#batch.length >= BUILDER_BATCH) {
|
|
163
|
+
this.#done.push(this.#batch.join(""));
|
|
164
|
+
this.#batch = [];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
toString() {
|
|
168
|
+
this.#done.push(this.#batch.join(""));
|
|
169
|
+
this.#batch = [];
|
|
170
|
+
return this.#done.join("");
|
|
171
|
+
}
|
|
172
|
+
}
|
|
55
173
|
/**
|
|
56
174
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
-
* as `credentialsIn` returns it) back in an error body
|
|
58
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
59
|
-
* `user:password` itself, and the
|
|
60
|
-
*
|
|
61
|
-
*
|
|
175
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
176
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
177
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
178
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
179
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
180
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
181
|
+
* strings (`redactSecrets`).
|
|
62
182
|
*/
|
|
63
183
|
export function echoedCredentialForms(userinfo) {
|
|
64
184
|
const colon = userinfo.indexOf(":");
|
|
65
|
-
if (colon < 0)
|
|
66
|
-
return [];
|
|
67
185
|
const decode = (part) => {
|
|
68
186
|
try {
|
|
69
187
|
return decodeURIComponent(part);
|
|
@@ -72,14 +190,15 @@ export function echoedCredentialForms(userinfo) {
|
|
|
72
190
|
return part;
|
|
73
191
|
}
|
|
74
192
|
};
|
|
75
|
-
const user = decode(userinfo.slice(0, colon));
|
|
76
|
-
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
-
|
|
193
|
+
const user = decode(colon < 0 ? userinfo : userinfo.slice(0, colon));
|
|
194
|
+
const password = colon < 0 ? "" : decode(userinfo.slice(colon + 1));
|
|
195
|
+
const secret = password === "" ? user : password;
|
|
196
|
+
if (secret === "")
|
|
78
197
|
return [];
|
|
79
198
|
const pair = `${user}:${password}`;
|
|
80
199
|
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
-
if (
|
|
82
|
-
forms.push(
|
|
200
|
+
if (secret.length >= 4)
|
|
201
|
+
forms.push(secret);
|
|
83
202
|
return forms;
|
|
84
203
|
}
|
|
85
204
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -186,6 +305,12 @@ export class CkanApiError extends CkanError {
|
|
|
186
305
|
detail;
|
|
187
306
|
url;
|
|
188
307
|
method;
|
|
308
|
+
/**
|
|
309
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
310
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
311
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
312
|
+
* for such a body nested too deeply to write again.
|
|
313
|
+
*/
|
|
189
314
|
body;
|
|
190
315
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
191
316
|
retries;
|
|
@@ -291,8 +416,21 @@ export function describeCkanError(error) {
|
|
|
291
416
|
* quotes and newlines escaped several levels deep.
|
|
292
417
|
*/
|
|
293
418
|
function solrReason(message) {
|
|
294
|
-
|
|
419
|
+
// Only a window after the first "[Reason: " is read: the pattern's lazy group, tried against
|
|
420
|
+
// `\\+n` at every step, is quadratic on a run of backslashes, and the message is uncut server
|
|
421
|
+
// text (80 KB took 2.4 s, past --timeout, and it grows with the square). The window is twice what a
|
|
422
|
+
// message shows; a reason that runs past it ends at the window's end.
|
|
423
|
+
const at = message.indexOf(REASON);
|
|
424
|
+
if (at === -1)
|
|
425
|
+
return undefined;
|
|
426
|
+
const end = at + REASON.length + 2 * MAX_MESSAGE_VALUE_LENGTH;
|
|
427
|
+
const window = message.slice(at, end);
|
|
428
|
+
const pattern = end < message.length ? SOLR_REASON_IN_CUT_WINDOW : SOLR_REASON;
|
|
429
|
+
const match = pattern.exec(window);
|
|
295
430
|
if (!match?.[1])
|
|
296
431
|
return undefined;
|
|
297
432
|
return match[1].replace(/\\+(['"])/g, "$1").trim();
|
|
298
433
|
}
|
|
434
|
+
const REASON = "[Reason: ";
|
|
435
|
+
const SOLR_REASON = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/;
|
|
436
|
+
const SOLR_REASON_IN_CUT_WINDOW = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\]|$)/;
|
|
@@ -48,6 +48,7 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
|
|
|
48
48
|
/**
|
|
49
49
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
50
50
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
51
|
-
* (connection errors, timeouts, malformed URLs).
|
|
51
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
52
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
52
53
|
*/
|
|
53
54
|
export declare const nodeHttpTransport: Transport;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// `http.createServer` in the test-suite.
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
|
-
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
10
|
+
import { CkanNetworkError, redactUrl, credentialsIn, redactCredentials, typedUrlCredentials } from "./errors.js";
|
|
11
11
|
/** The message for a body over the size cap, naming the option on both sides. */
|
|
12
12
|
export function sizeLimitMessage(maxBytes) {
|
|
13
13
|
return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
|
|
@@ -17,10 +17,23 @@ export function sizeLimitMessage(maxBytes) {
|
|
|
17
17
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
18
18
|
*/
|
|
19
19
|
export const MAX_TIMEOUT_MS = 2_147_483_647;
|
|
20
|
+
/** The most received chunks the default transport keeps unjoined. */
|
|
21
|
+
const COALESCE_CHUNKS = 1024;
|
|
22
|
+
/** The bytes of unjoined chunks at which the default transport joins them into one Buffer. */
|
|
23
|
+
const COALESCE_BYTES = 64 * 1024;
|
|
24
|
+
/**
|
|
25
|
+
* `url` for a message, its userinfo redacted; one typed without its scheme (`user:pw@host`,
|
|
26
|
+
* `TOKEN:@host`) or with a mistyped one (`http//user:pw@host`) is read as a URL all the same
|
|
27
|
+
* (`typedUrlCredentials`), since `redactUrl` finds a userinfo only after a scheme.
|
|
28
|
+
*/
|
|
29
|
+
function shownUrl(url) {
|
|
30
|
+
return credentialsIn(url).length > 0 ? redactUrl(url) : redactCredentials(url, typedUrlCredentials(url));
|
|
31
|
+
}
|
|
20
32
|
/**
|
|
21
33
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
22
34
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
23
|
-
* (connection errors, timeouts, malformed URLs).
|
|
35
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
36
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
24
37
|
*/
|
|
25
38
|
export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
26
39
|
let url;
|
|
@@ -28,14 +41,17 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
28
41
|
url = new URL(request.url);
|
|
29
42
|
}
|
|
30
43
|
catch {
|
|
31
|
-
reject(new CkanNetworkError(`Invalid URL: ${
|
|
44
|
+
reject(new CkanNetworkError(`Invalid URL: ${shownUrl(request.url)}`));
|
|
32
45
|
return;
|
|
33
46
|
}
|
|
34
47
|
// Only http/https are supported. Reject anything else up front with a clear,
|
|
35
48
|
// typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
|
|
36
49
|
// (and so this never reaches the file:/ftp:/etc. drivers).
|
|
37
50
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
38
|
-
|
|
51
|
+
// `user:password@host` without a scheme parses with the user name as the scheme: name a
|
|
52
|
+
// scheme only when the value spells one (`<scheme>://`).
|
|
53
|
+
const named = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(request.url) ? ` "${url.protocol}"` : "";
|
|
54
|
+
reject(new CkanNetworkError(`Unsupported protocol${named} in URL: ${shownUrl(request.url)}`));
|
|
39
55
|
return;
|
|
40
56
|
}
|
|
41
57
|
const isHttps = url.protocol === "https:";
|
|
@@ -62,7 +78,12 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
62
78
|
method: request.method,
|
|
63
79
|
headers: request.headers,
|
|
64
80
|
}, (res) => {
|
|
81
|
+
// Small chunks are joined as they arrive (`pending`, at most COALESCE_CHUNKS of them
|
|
82
|
+
// or COALESCE_BYTES), so the Buffers kept follow the bytes, not the number of chunks
|
|
83
|
+
// a server chooses to send: a Buffer per one-byte chunk held ~210 bytes of heap each.
|
|
65
84
|
const chunks = [];
|
|
85
|
+
let pending = [];
|
|
86
|
+
let pendingBytes = 0;
|
|
66
87
|
let received = 0;
|
|
67
88
|
let aborted = false;
|
|
68
89
|
res.on("data", (chunk) => {
|
|
@@ -75,7 +96,13 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
75
96
|
settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
|
|
76
97
|
return;
|
|
77
98
|
}
|
|
78
|
-
|
|
99
|
+
pending.push(chunk);
|
|
100
|
+
pendingBytes += chunk.length;
|
|
101
|
+
if (pending.length >= COALESCE_CHUNKS || pendingBytes >= COALESCE_BYTES) {
|
|
102
|
+
chunks.push(pending.length === 1 ? chunk : Buffer.concat(pending, pendingBytes));
|
|
103
|
+
pending = [];
|
|
104
|
+
pendingBytes = 0;
|
|
105
|
+
}
|
|
79
106
|
});
|
|
80
107
|
res.on("end", () => {
|
|
81
108
|
if (aborted)
|
|
@@ -83,7 +110,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
83
110
|
settleResolve({
|
|
84
111
|
status: res.statusCode ?? 0,
|
|
85
112
|
headers: res.headers,
|
|
86
|
-
body: Buffer.concat(chunks),
|
|
113
|
+
body: Buffer.concat([...chunks, ...pending]),
|
|
87
114
|
});
|
|
88
115
|
});
|
|
89
116
|
res.on("error", (err) => {
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
1
|
+
export { actionRequest, ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, packageSearchQuery, siteRoot } from "./client.js";
|
|
2
2
|
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
3
3
|
export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
@@ -12,3 +12,4 @@ export { PORTALS } from "./portals-list.js";
|
|
|
12
12
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
13
13
|
export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
|
|
14
14
|
export * from "./types.js";
|
|
15
|
+
export * from "./operations.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
|
-
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
|
+
export { actionRequest, ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, packageSearchQuery, siteRoot } from "./client.js";
|
|
3
3
|
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
@@ -8,3 +8,4 @@ export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, norma
|
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
10
10
|
export * from "./types.js";
|
|
11
|
+
export * from "./operations.js";
|