@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/README.md +21 -7
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
  15. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
  16. package/dist/src/cli/commands/catalogue.js +12 -11
  17. package/dist/src/cli/commands/portal.js +28 -20
  18. package/dist/src/cli/io.d.ts +16 -0
  19. package/dist/src/cli/io.js +55 -1
  20. package/dist/src/cli/output.d.ts +26 -0
  21. package/dist/src/cli/output.js +151 -0
  22. package/dist/src/cli/run.d.ts +0 -7
  23. package/dist/src/cli/run.js +80 -17
  24. package/dist/src/cli/shared.d.ts +14 -25
  25. package/dist/src/cli/shared.js +40 -102
  26. package/package.json +1 -1
@@ -9,20 +9,60 @@ export declare function redactUrl(url: string): string;
9
9
  /**
10
10
  * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
11
11
  * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
12
- * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
12
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
13
+ * slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
13
14
  * search text or a User-Agent as often as a credential, and the base URL always has a
14
15
  * scheme. It works on URLs that don't parse too: the userinfo is everything between
15
16
  * `://` and the last `@` before the host. Used to redact those exact strings from text
16
17
  * that echoes the value (usage errors, help), whatever characters the password contains.
17
18
  */
18
19
  export declare function credentialsIn(value: string): string[];
20
+ /**
21
+ * The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
22
+ * typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
23
+ * (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
24
+ * (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
25
+ * first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
26
+ * doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
27
+ * it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
28
+ * other text a bare `a/b@c` is no credential.
29
+ */
30
+ export declare function typedUrlCredentials(value: string): string[];
31
+ /**
32
+ * `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
33
+ * becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
34
+ * so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
35
+ * whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
36
+ * `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
37
+ * and the search goes on right after each `://`, not after the authority it read: a URL glued
38
+ * onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
39
+ * authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
40
+ * and every character is read a bounded number of times: by the authority loop and `indexOf`
41
+ * forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
42
+ * word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
43
+ */
44
+ export declare function redactUrlUserinfo(text: string): string;
45
+ /**
46
+ * A text put together from many pieces (slices of a server answer and their replacements),
47
+ * joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
48
+ * not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
49
+ * piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
50
+ * either way.
51
+ */
52
+ export declare class TextBuilder {
53
+ #private;
54
+ push(...pieces: string[]): void;
55
+ toString(): string;
56
+ }
19
57
  /**
20
58
  * The forms in which a server may echo the credentials of a userinfo (`user:password`,
21
- * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
22
- * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
23
- * `user:password` itself, and the password alone when it is at least 4 characters long.
24
- * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
25
- * are replaced as exact strings (`redactSecrets`).
59
+ * as `credentialsIn` returns it) back in an error body, in this order: the
60
+ * `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
61
+ * sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
62
+ * characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
63
+ * `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
64
+ * for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
65
+ * strings (`redactSecrets`).
26
66
  */
27
67
  export declare function echoedCredentialForms(userinfo: string): string[];
28
68
  /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
@@ -51,6 +91,18 @@ export declare function cutForMessage(text: string): string;
51
91
  * Text no longer than `max` is returned as it is; the caller marks a cut.
52
92
  */
53
93
  export declare function cutText(text: string, max: number): string;
94
+ /**
95
+ * Quote a value for an error message, as `JSON.stringify` does, and also escape what it
96
+ * leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
97
+ * format character (bidi overrides, zero-width characters) and the line and paragraph
98
+ * separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
99
+ * messages, and it can come from a pipeline or from the portal; an override in it would
100
+ * reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
101
+ * was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
102
+ * "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
103
+ * The quoted part stays a valid JSON string.
104
+ */
105
+ export declare function quoteValue(value: string): string;
54
106
  /**
55
107
  * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
56
108
  * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
@@ -71,6 +123,12 @@ export declare class CkanApiError extends CkanError {
71
123
  readonly detail: string | undefined;
72
124
  readonly url: string;
73
125
  readonly method: string;
126
+ /**
127
+ * The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
128
+ * whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
129
+ * written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
130
+ * for such a body nested too deeply to write again.
131
+ */
74
132
  readonly body: string;
75
133
  /** How many times the engine retried the request before giving up (0 when it did not). */
76
134
  readonly retries: number;
@@ -96,16 +154,19 @@ export declare class CkanApiError extends CkanError {
96
154
  /**
97
155
  * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
98
156
  * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
99
- * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
157
+ * message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
100
158
  * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
101
159
  * or undefined when the error carries none.
102
160
  */
103
161
  export declare class CkanActionError extends CkanError {
104
162
  readonly action: string;
105
163
  readonly errorType: string | undefined;
164
+ /** The URL that answered, userinfo redacted; empty when the error was built without one. */
165
+ readonly url: string;
106
166
  constructor(message: string, args: {
107
167
  action: string;
108
168
  errorType?: string | undefined;
169
+ url?: string | undefined;
109
170
  });
110
171
  /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
111
172
  get isNotFound(): boolean;
@@ -27,14 +27,17 @@ export function redactUrl(url) {
27
27
  /**
28
28
  * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
29
29
  * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
30
- * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
30
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
31
+ * slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
31
32
  * search text or a User-Agent as often as a credential, and the base URL always has a
32
33
  * scheme. It works on URLs that don't parse too: the userinfo is everything between
33
34
  * `://` and the last `@` before the host. Used to redact those exact strings from text
34
35
  * that echoes the value (usage errors, help), whatever characters the password contains.
35
36
  */
36
37
  export function credentialsIn(value) {
37
- const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
+ // WHATWG skips any number of "/" and "\" after a special scheme (`http:///u:pw@h`,
39
+ // `https:\\u:pw@h` and `http:u:pw@h` all send `u:pw`), so its userinfo starts after them.
40
+ const scheme = SPECIAL_SCHEME_START.exec(value) ?? /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
41
  if (scheme === null)
39
42
  return [];
40
43
  const rest = value.slice(scheme[0].length);
@@ -47,23 +50,138 @@ export function credentialsIn(value) {
47
50
  // Doesn't parse: the password may hold "/", "?", "#" or spaces.
48
51
  }
49
52
  // In a URL that parses, the userinfo ends at the last "@" of the authority (before the
50
- // first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
51
- const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
53
+ // first "/", "?" or "#", and for a special scheme such as http(s) a "\", where WHATWG ends
54
+ // it too: `https://u:pw@a.example\@b.example` sends `u:pw` to a.example); in one that doesn't,
55
+ // at the last "@" of the value.
56
+ const stop = SPECIAL_SCHEME.test(scheme[0]) ? /[/?#\\]|$/ : /[/?#]|$/;
57
+ const authority = parses ? rest.slice(0, rest.search(stop)) : rest;
52
58
  const end = authority.lastIndexOf("@");
53
59
  return end > 0 ? [rest.slice(0, end)] : [];
54
60
  }
61
+ /**
62
+ * The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
63
+ * typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
64
+ * (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
65
+ * (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
66
+ * first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
67
+ * doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
68
+ * it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
69
+ * other text a bare `a/b@c` is no credential.
70
+ */
71
+ export function typedUrlCredentials(value) {
72
+ if (value === "" || credentialsIn(value).length > 0 || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
73
+ return credentialsIn(value);
74
+ const prefixed = credentialsIn(`http://${value}`);
75
+ if (prefixed.length > 0)
76
+ return prefixed;
77
+ const slashes = /[/\\]+/.exec(value);
78
+ const at = value.lastIndexOf("@");
79
+ if (slashes === null || slashes.index + slashes[0].length >= at)
80
+ return [];
81
+ return [value.slice(slashes.index + slashes[0].length, at)];
82
+ }
83
+ /** The schemes WHATWG parses as special (`ftp`, `file`, `http(s)`, `ws(s)`): their authority also ends at a backslash. */
84
+ const SPECIAL_SCHEME = /^(?:ftp|file|https?|wss?):[/\\]*$/i;
85
+ /** A special scheme at the start of a value, with the slashes and backslashes WHATWG skips after it. */
86
+ const SPECIAL_SCHEME_START = /^(?:ftp|file|https?|wss?):[/\\]*/i;
87
+ /** A scheme character (RFC 3986: letters, digits, `+`, `.`, `-`). */
88
+ function isSchemeChar(code) {
89
+ return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39) || code === 0x2b || code === 0x2e || code === 0x2d;
90
+ }
91
+ /** An ASCII letter, with which a scheme starts. */
92
+ function isLetter(code) {
93
+ return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);
94
+ }
95
+ /**
96
+ * Where the scheme of the `://` at `at` starts: the first letter of the run of scheme characters
97
+ * before it, looking back no further than `from`; -1 when there is none (`1://`, ` ://`). A run
98
+ * led by other scheme characters (`-https://`) starts at its first letter.
99
+ */
100
+ function schemeStart(text, at, from) {
101
+ let start = at;
102
+ while (start > from && isSchemeChar(text.charCodeAt(start - 1)))
103
+ start--;
104
+ while (start < at && !isLetter(text.charCodeAt(start)))
105
+ start++;
106
+ return start < at ? start : -1;
107
+ }
108
+ /**
109
+ * `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
110
+ * becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
111
+ * so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
112
+ * whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
113
+ * `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
114
+ * and the search goes on right after each `://`, not after the authority it read: a URL glued
115
+ * onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
116
+ * authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
117
+ * and every character is read a bounded number of times: by the authority loop and `indexOf`
118
+ * forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
119
+ * word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
120
+ */
121
+ export function redactUrlUserinfo(text) {
122
+ const parts = new TextBuilder();
123
+ let copied = 0;
124
+ let from = 0;
125
+ for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", from)) {
126
+ const begin = at + 3;
127
+ let end = begin;
128
+ let lastAt = -1;
129
+ for (; end < text.length; end++) {
130
+ const c = text.charCodeAt(end);
131
+ if (c === 0x2f || c === 0x3f || c === 0x23 || c === 0x27 || /\s/.test(text[end]))
132
+ break;
133
+ if (c === 0x40)
134
+ lastAt = end;
135
+ }
136
+ if (lastAt !== -1 && schemeStart(text, at, from) !== -1) {
137
+ parts.push(text.slice(copied, begin), "***");
138
+ copied = lastAt;
139
+ }
140
+ from = begin;
141
+ }
142
+ if (copied === 0)
143
+ return text;
144
+ parts.push(text.slice(copied));
145
+ return parts.toString();
146
+ }
147
+ /** How many pieces a TextBuilder joins at a time. */
148
+ const BUILDER_BATCH = 16_384;
149
+ /**
150
+ * A text put together from many pieces (slices of a server answer and their replacements),
151
+ * joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
152
+ * not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
153
+ * piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
154
+ * either way.
155
+ */
156
+ export class TextBuilder {
157
+ #done = [];
158
+ #batch = [];
159
+ push(...pieces) {
160
+ for (const piece of pieces)
161
+ this.#batch.push(piece);
162
+ if (this.#batch.length >= BUILDER_BATCH) {
163
+ this.#done.push(this.#batch.join(""));
164
+ this.#batch = [];
165
+ }
166
+ }
167
+ toString() {
168
+ this.#done.push(this.#batch.join(""));
169
+ this.#batch = [];
170
+ return this.#done.join("");
171
+ }
172
+ }
55
173
  /**
56
174
  * The forms in which a server may echo the credentials of a userinfo (`user:password`,
57
- * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
58
- * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
59
- * `user:password` itself, and the password alone when it is at least 4 characters long.
60
- * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
61
- * are replaced as exact strings (`redactSecrets`).
175
+ * as `credentialsIn` returns it) back in an error body, in this order: the
176
+ * `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
177
+ * sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
178
+ * characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
179
+ * `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
180
+ * for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
181
+ * strings (`redactSecrets`).
62
182
  */
63
183
  export function echoedCredentialForms(userinfo) {
64
184
  const colon = userinfo.indexOf(":");
65
- if (colon < 0)
66
- return [];
67
185
  const decode = (part) => {
68
186
  try {
69
187
  return decodeURIComponent(part);
@@ -72,14 +190,15 @@ export function echoedCredentialForms(userinfo) {
72
190
  return part;
73
191
  }
74
192
  };
75
- const user = decode(userinfo.slice(0, colon));
76
- const password = decode(userinfo.slice(colon + 1));
77
- if (password === "")
193
+ const user = decode(colon < 0 ? userinfo : userinfo.slice(0, colon));
194
+ const password = colon < 0 ? "" : decode(userinfo.slice(colon + 1));
195
+ const secret = password === "" ? user : password;
196
+ if (secret === "")
78
197
  return [];
79
198
  const pair = `${user}:${password}`;
80
199
  const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
81
- if (password.length >= 4)
82
- forms.push(password);
200
+ if (secret.length >= 4)
201
+ forms.push(secret);
83
202
  return forms;
84
203
  }
85
204
  /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
@@ -134,6 +253,35 @@ export function cutText(text, max) {
134
253
  function isHighSurrogate(c) {
135
254
  return c >= 0xd800 && c <= 0xdbff;
136
255
  }
256
+ /** Unicode General Category Cf (format): bidi controls, zero-width characters, U+00AD, U+FEFF, … */
257
+ const FORMAT_CHAR = /\p{Cf}/u;
258
+ /**
259
+ * Quote a value for an error message, as `JSON.stringify` does, and also escape what it
260
+ * leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
261
+ * format character (bidi overrides, zero-width characters) and the line and paragraph
262
+ * separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
263
+ * messages, and it can come from a pipeline or from the portal; an override in it would
264
+ * reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
265
+ * was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
266
+ * "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
267
+ * The quoted part stays a valid JSON string.
268
+ */
269
+ export function quoteValue(value) {
270
+ const cut = value.length > MAX_MESSAGE_VALUE_LENGTH;
271
+ let out = "";
272
+ for (const ch of JSON.stringify(cut ? cutText(value, MAX_MESSAGE_VALUE_LENGTH) : value)) {
273
+ const n = ch.codePointAt(0) ?? 0;
274
+ if ((n >= 0x7f && n <= 0x9f) || n === 0x2028 || n === 0x2029 || FORMAT_CHAR.test(ch)) {
275
+ // One escape per UTF-16 unit, so a character above U+FFFF stays valid JSON.
276
+ for (let i = 0; i < ch.length; i++)
277
+ out += `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`;
278
+ }
279
+ else {
280
+ out += ch;
281
+ }
282
+ }
283
+ return cut ? `${out}…` : out;
284
+ }
137
285
  /**
138
286
  * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
139
287
  * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
@@ -157,6 +305,12 @@ export class CkanApiError extends CkanError {
157
305
  detail;
158
306
  url;
159
307
  method;
308
+ /**
309
+ * The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
310
+ * whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
311
+ * written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
312
+ * for such a body nested too deeply to write again.
313
+ */
160
314
  body;
161
315
  /** How many times the engine retried the request before giving up (0 when it did not). */
162
316
  retries;
@@ -201,17 +355,20 @@ export class CkanApiError extends CkanError {
201
355
  /**
202
356
  * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
203
357
  * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
204
- * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
358
+ * message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
205
359
  * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
206
360
  * or undefined when the error carries none.
207
361
  */
208
362
  export class CkanActionError extends CkanError {
209
363
  action;
210
364
  errorType;
365
+ /** The URL that answered, userinfo redacted; empty when the error was built without one. */
366
+ url;
211
367
  constructor(message, args) {
212
368
  super(message);
213
369
  this.action = args.action;
214
370
  this.errorType = args.errorType;
371
+ this.url = args.url === undefined ? "" : redactUrl(args.url);
215
372
  }
216
373
  /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
217
374
  get isNotFound() {
@@ -259,8 +416,21 @@ export function describeCkanError(error) {
259
416
  * quotes and newlines escaped several levels deep.
260
417
  */
261
418
  function solrReason(message) {
262
- const match = /\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/.exec(message);
419
+ // Only a window after the first "[Reason: " is read: the pattern's lazy group, tried against
420
+ // `\\+n` at every step, is quadratic on a run of backslashes, and the message is uncut server
421
+ // text (80 KB took 2.4 s, past --timeout, and it grows with the square). The window is twice what a
422
+ // message shows; a reason that runs past it ends at the window's end.
423
+ const at = message.indexOf(REASON);
424
+ if (at === -1)
425
+ return undefined;
426
+ const end = at + REASON.length + 2 * MAX_MESSAGE_VALUE_LENGTH;
427
+ const window = message.slice(at, end);
428
+ const pattern = end < message.length ? SOLR_REASON_IN_CUT_WINDOW : SOLR_REASON;
429
+ const match = pattern.exec(window);
263
430
  if (!match?.[1])
264
431
  return undefined;
265
432
  return match[1].replace(/\\+(['"])/g, "$1").trim();
266
433
  }
434
+ const REASON = "[Reason: ";
435
+ const SOLR_REASON = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/;
436
+ const SOLR_REASON_IN_CUT_WINDOW = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\]|$)/;
@@ -48,6 +48,7 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
48
48
  /**
49
49
  * Default transport. Resolves with the raw response (including non-2xx) — status
50
50
  * interpretation is the client's job. Rejects only on transport-level failures
51
- * (connection errors, timeouts, malformed URLs).
51
+ * (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
52
+ * number of chunks a server sends it in: small chunks are joined as they arrive.
52
53
  */
53
54
  export declare const nodeHttpTransport: Transport;
@@ -7,7 +7,7 @@
7
7
  // `http.createServer` in the test-suite.
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
- import { CkanNetworkError, redactUrl } from "./errors.js";
10
+ import { CkanNetworkError, redactUrl, credentialsIn, redactCredentials, typedUrlCredentials } from "./errors.js";
11
11
  /** The message for a body over the size cap, naming the option on both sides. */
12
12
  export function sizeLimitMessage(maxBytes) {
13
13
  return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
@@ -17,10 +17,23 @@ export function sizeLimitMessage(maxBytes) {
17
17
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
18
18
  */
19
19
  export const MAX_TIMEOUT_MS = 2_147_483_647;
20
+ /** The most received chunks the default transport keeps unjoined. */
21
+ const COALESCE_CHUNKS = 1024;
22
+ /** The bytes of unjoined chunks at which the default transport joins them into one Buffer. */
23
+ const COALESCE_BYTES = 64 * 1024;
24
+ /**
25
+ * `url` for a message, its userinfo redacted; one typed without its scheme (`user:pw@host`,
26
+ * `TOKEN:@host`) or with a mistyped one (`http//user:pw@host`) is read as a URL all the same
27
+ * (`typedUrlCredentials`), since `redactUrl` finds a userinfo only after a scheme.
28
+ */
29
+ function shownUrl(url) {
30
+ return credentialsIn(url).length > 0 ? redactUrl(url) : redactCredentials(url, typedUrlCredentials(url));
31
+ }
20
32
  /**
21
33
  * Default transport. Resolves with the raw response (including non-2xx) — status
22
34
  * interpretation is the client's job. Rejects only on transport-level failures
23
- * (connection errors, timeouts, malformed URLs).
35
+ * (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
36
+ * number of chunks a server sends it in: small chunks are joined as they arrive.
24
37
  */
25
38
  export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
26
39
  let url;
@@ -28,14 +41,17 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
28
41
  url = new URL(request.url);
29
42
  }
30
43
  catch {
31
- reject(new CkanNetworkError(`Invalid URL: ${redactUrl(request.url)}`));
44
+ reject(new CkanNetworkError(`Invalid URL: ${shownUrl(request.url)}`));
32
45
  return;
33
46
  }
34
47
  // Only http/https are supported. Reject anything else up front with a clear,
35
48
  // typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
36
49
  // (and so this never reaches the file:/ftp:/etc. drivers).
37
50
  if (url.protocol !== "http:" && url.protocol !== "https:") {
38
- reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${redactUrl(request.url)}`));
51
+ // `user:password@host` without a scheme parses with the user name as the scheme: name a
52
+ // scheme only when the value spells one (`<scheme>://`).
53
+ const named = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(request.url) ? ` "${url.protocol}"` : "";
54
+ reject(new CkanNetworkError(`Unsupported protocol${named} in URL: ${shownUrl(request.url)}`));
39
55
  return;
40
56
  }
41
57
  const isHttps = url.protocol === "https:";
@@ -62,7 +78,12 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
62
78
  method: request.method,
63
79
  headers: request.headers,
64
80
  }, (res) => {
81
+ // Small chunks are joined as they arrive (`pending`, at most COALESCE_CHUNKS of them
82
+ // or COALESCE_BYTES), so the Buffers kept follow the bytes, not the number of chunks
83
+ // a server chooses to send: a Buffer per one-byte chunk held ~210 bytes of heap each.
65
84
  const chunks = [];
85
+ let pending = [];
86
+ let pendingBytes = 0;
66
87
  let received = 0;
67
88
  let aborted = false;
68
89
  res.on("data", (chunk) => {
@@ -75,7 +96,13 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
75
96
  settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
76
97
  return;
77
98
  }
78
- chunks.push(chunk);
99
+ pending.push(chunk);
100
+ pendingBytes += chunk.length;
101
+ if (pending.length >= COALESCE_CHUNKS || pendingBytes >= COALESCE_BYTES) {
102
+ chunks.push(pending.length === 1 ? chunk : Buffer.concat(pending, pendingBytes));
103
+ pending = [];
104
+ pendingBytes = 0;
105
+ }
79
106
  });
80
107
  res.on("end", () => {
81
108
  if (aborted)
@@ -83,7 +110,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
83
110
  settleResolve({
84
111
  status: res.statusCode ?? 0,
85
112
  headers: res.headers,
86
- body: Buffer.concat(chunks),
113
+ body: Buffer.concat([...chunks, ...pending]),
87
114
  });
88
115
  });
89
116
  res.on("error", (err) => {
@@ -5,8 +5,8 @@ export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
- export { assertValid, normalizeText } from "./validate.js";
8
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
+ export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
12
12
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -3,8 +3,8 @@ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot }
3
3
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
- export { assertValid, normalizeText } from "./validate.js";
6
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
+ export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
10
10
  export * from "./types.js";
@@ -12,7 +12,10 @@ export declare function findPortal(idOrUrl: string, list: readonly Portal[]): Po
12
12
  /** The outcome of `checkPortal`. */
13
13
  export interface PortalCheck {
14
14
  working: boolean;
15
- /** A short reason when not working, e.g. `HTTP 404`, `host not found`. */
15
+ /**
16
+ * A short reason when not working, e.g. `HTTP 404`, `host not found`, `not JSON (text/html)`,
17
+ * `invalid JSON`, `inexact number` (valid JSON with a number a double can't carry), `empty answer`.
18
+ */
16
19
  problem: string | null;
17
20
  datasets: number | null;
18
21
  /** From `status_show`, which some portals lock (Berlin): then null. */
@@ -94,13 +94,21 @@ function problemOf(err) {
94
94
  }
95
95
  const message = err instanceof Error ? err.message : String(err);
96
96
  if (err instanceof CkanParseError) {
97
- const got = /but got (.+)$/.exec(message);
97
+ // The engine's "GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "<type>")."
98
+ const got = /\(expected JSON, got Content-Type "(.*)"\)\.$/.exec(message);
98
99
  if (got)
99
100
  return `not JSON (${got[1]})`;
100
- if (/not a CKAN Action API response/.test(message))
101
+ if (/: the answer is not a CKAN Action API response; /.test(message))
101
102
  return "not a CKAN Action API";
102
- if (/^Unexpected response shape/.test(message))
103
+ if (/: unexpected response shape: /.test(message))
103
104
  return "unexpected response shape";
105
+ // Valid JSON whose number a double can't carry, or no body at all: not "invalid JSON".
106
+ if (/: the number .* cannot be represented exactly\.$/.test(message))
107
+ return "inexact number";
108
+ if (/: empty response body\.$/.test(message))
109
+ return "empty answer";
110
+ if (/: unsupported response charset /.test(message))
111
+ return "unsupported charset";
104
112
  return "invalid JSON";
105
113
  }
106
114
  if (err instanceof CkanNetworkError) {