@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +80 -17
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
|
@@ -9,20 +9,60 @@ export declare function redactUrl(url: string): string;
|
|
|
9
9
|
/**
|
|
10
10
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
11
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
13
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
13
14
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
15
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
16
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
16
17
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
17
18
|
*/
|
|
18
19
|
export declare function credentialsIn(value: string): string[];
|
|
20
|
+
/**
|
|
21
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
22
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
23
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
24
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
25
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
26
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
27
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
28
|
+
* other text a bare `a/b@c` is no credential.
|
|
29
|
+
*/
|
|
30
|
+
export declare function typedUrlCredentials(value: string): string[];
|
|
31
|
+
/**
|
|
32
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
33
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
34
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
35
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
36
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
37
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
38
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
39
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
40
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
41
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
42
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
43
|
+
*/
|
|
44
|
+
export declare function redactUrlUserinfo(text: string): string;
|
|
45
|
+
/**
|
|
46
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
47
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
48
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
49
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
50
|
+
* either way.
|
|
51
|
+
*/
|
|
52
|
+
export declare class TextBuilder {
|
|
53
|
+
#private;
|
|
54
|
+
push(...pieces: string[]): void;
|
|
55
|
+
toString(): string;
|
|
56
|
+
}
|
|
19
57
|
/**
|
|
20
58
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
-
* as `credentialsIn` returns it) back in an error body
|
|
22
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
23
|
-
* `user:password` itself, and the
|
|
24
|
-
*
|
|
25
|
-
*
|
|
59
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
60
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
61
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
62
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
63
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
64
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
65
|
+
* strings (`redactSecrets`).
|
|
26
66
|
*/
|
|
27
67
|
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
68
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -51,6 +91,18 @@ export declare function cutForMessage(text: string): string;
|
|
|
51
91
|
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
52
92
|
*/
|
|
53
93
|
export declare function cutText(text: string, max: number): string;
|
|
94
|
+
/**
|
|
95
|
+
* Quote a value for an error message, as `JSON.stringify` does, and also escape what it
|
|
96
|
+
* leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
|
|
97
|
+
* format character (bidi overrides, zero-width characters) and the line and paragraph
|
|
98
|
+
* separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
|
|
99
|
+
* messages, and it can come from a pipeline or from the portal; an override in it would
|
|
100
|
+
* reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
|
|
101
|
+
* was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
|
|
102
|
+
* "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
|
|
103
|
+
* The quoted part stays a valid JSON string.
|
|
104
|
+
*/
|
|
105
|
+
export declare function quoteValue(value: string): string;
|
|
54
106
|
/**
|
|
55
107
|
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
56
108
|
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
@@ -71,6 +123,12 @@ export declare class CkanApiError extends CkanError {
|
|
|
71
123
|
readonly detail: string | undefined;
|
|
72
124
|
readonly url: string;
|
|
73
125
|
readonly method: string;
|
|
126
|
+
/**
|
|
127
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
128
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
129
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
130
|
+
* for such a body nested too deeply to write again.
|
|
131
|
+
*/
|
|
74
132
|
readonly body: string;
|
|
75
133
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
76
134
|
readonly retries: number;
|
|
@@ -96,16 +154,19 @@ export declare class CkanApiError extends CkanError {
|
|
|
96
154
|
/**
|
|
97
155
|
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
98
156
|
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
99
|
-
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
157
|
+
* message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
100
158
|
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
101
159
|
* or undefined when the error carries none.
|
|
102
160
|
*/
|
|
103
161
|
export declare class CkanActionError extends CkanError {
|
|
104
162
|
readonly action: string;
|
|
105
163
|
readonly errorType: string | undefined;
|
|
164
|
+
/** The URL that answered, userinfo redacted; empty when the error was built without one. */
|
|
165
|
+
readonly url: string;
|
|
106
166
|
constructor(message: string, args: {
|
|
107
167
|
action: string;
|
|
108
168
|
errorType?: string | undefined;
|
|
169
|
+
url?: string | undefined;
|
|
109
170
|
});
|
|
110
171
|
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
111
172
|
get isNotFound(): boolean;
|
|
@@ -27,14 +27,17 @@ export function redactUrl(url) {
|
|
|
27
27
|
/**
|
|
28
28
|
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
29
|
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
-
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`, or a special one such as `http:` with any
|
|
31
|
+
* slashes or backslashes after it, as WHATWG reads it) counts: a bare `a:b@c` is a dataset id, a
|
|
31
32
|
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
33
|
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
34
|
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
34
35
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
35
36
|
*/
|
|
36
37
|
export function credentialsIn(value) {
|
|
37
|
-
|
|
38
|
+
// WHATWG skips any number of "/" and "\" after a special scheme (`http:///u:pw@h`,
|
|
39
|
+
// `https:\\u:pw@h` and `http:u:pw@h` all send `u:pw`), so its userinfo starts after them.
|
|
40
|
+
const scheme = SPECIAL_SCHEME_START.exec(value) ?? /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
41
|
if (scheme === null)
|
|
39
42
|
return [];
|
|
40
43
|
const rest = value.slice(scheme[0].length);
|
|
@@ -47,23 +50,138 @@ export function credentialsIn(value) {
|
|
|
47
50
|
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
48
51
|
}
|
|
49
52
|
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
50
|
-
// first "/", "?" or "#"
|
|
51
|
-
|
|
53
|
+
// first "/", "?" or "#", and for a special scheme such as http(s) a "\", where WHATWG ends
|
|
54
|
+
// it too: `https://u:pw@a.example\@b.example` sends `u:pw` to a.example); in one that doesn't,
|
|
55
|
+
// at the last "@" of the value.
|
|
56
|
+
const stop = SPECIAL_SCHEME.test(scheme[0]) ? /[/?#\\]|$/ : /[/?#]|$/;
|
|
57
|
+
const authority = parses ? rest.slice(0, rest.search(stop)) : rest;
|
|
52
58
|
const end = authority.lastIndexOf("@");
|
|
53
59
|
return end > 0 ? [rest.slice(0, end)] : [];
|
|
54
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* The userinfo of a value meant as a URL (a base URL, a request URL), however its scheme is
|
|
63
|
+
* typed: as `credentialsIn` reads it when the value has a scheme; else as if it had `http://`
|
|
64
|
+
* (`user:password@host`, `TOKEN:@host`); else, for a scheme mistyped before slashes
|
|
65
|
+
* (`http//user:pw@host` with the colon left out, `http/…`, `htp:/…`), everything between the
|
|
66
|
+
* first run of slashes or backslashes and the last `@`, as `credentialsIn` reads a URL that
|
|
67
|
+
* doesn't parse. Such a value is refused, but the refusal (commander's, the transport's) quotes
|
|
68
|
+
* it, so its userinfo has to be found to be redacted. Only for a value known to be a URL: in
|
|
69
|
+
* other text a bare `a/b@c` is no credential.
|
|
70
|
+
*/
|
|
71
|
+
export function typedUrlCredentials(value) {
|
|
72
|
+
if (value === "" || credentialsIn(value).length > 0 || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
|
|
73
|
+
return credentialsIn(value);
|
|
74
|
+
const prefixed = credentialsIn(`http://${value}`);
|
|
75
|
+
if (prefixed.length > 0)
|
|
76
|
+
return prefixed;
|
|
77
|
+
const slashes = /[/\\]+/.exec(value);
|
|
78
|
+
const at = value.lastIndexOf("@");
|
|
79
|
+
if (slashes === null || slashes.index + slashes[0].length >= at)
|
|
80
|
+
return [];
|
|
81
|
+
return [value.slice(slashes.index + slashes[0].length, at)];
|
|
82
|
+
}
|
|
83
|
+
/** The schemes WHATWG parses as special (`ftp`, `file`, `http(s)`, `ws(s)`): their authority also ends at a backslash. */
|
|
84
|
+
const SPECIAL_SCHEME = /^(?:ftp|file|https?|wss?):[/\\]*$/i;
|
|
85
|
+
/** A special scheme at the start of a value, with the slashes and backslashes WHATWG skips after it. */
|
|
86
|
+
const SPECIAL_SCHEME_START = /^(?:ftp|file|https?|wss?):[/\\]*/i;
|
|
87
|
+
/** A scheme character (RFC 3986: letters, digits, `+`, `.`, `-`). */
|
|
88
|
+
function isSchemeChar(code) {
|
|
89
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || (code >= 0x30 && code <= 0x39) || code === 0x2b || code === 0x2e || code === 0x2d;
|
|
90
|
+
}
|
|
91
|
+
/** An ASCII letter, with which a scheme starts. */
|
|
92
|
+
function isLetter(code) {
|
|
93
|
+
return (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Where the scheme of the `://` at `at` starts: the first letter of the run of scheme characters
|
|
97
|
+
* before it, looking back no further than `from`; -1 when there is none (`1://`, ` ://`). A run
|
|
98
|
+
* led by other scheme characters (`-https://`) starts at its first letter.
|
|
99
|
+
*/
|
|
100
|
+
function schemeStart(text, at, from) {
|
|
101
|
+
let start = at;
|
|
102
|
+
while (start > from && isSchemeChar(text.charCodeAt(start - 1)))
|
|
103
|
+
start--;
|
|
104
|
+
while (start < at && !isLetter(text.charCodeAt(start)))
|
|
105
|
+
start++;
|
|
106
|
+
return start < at ? start : -1;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* `text` with the userinfo of every URL in it replaced by `***` (`https://user:secret@host`
|
|
110
|
+
* becomes `https://***@host`): the pattern backstop for text no exact secret matches. Text-based,
|
|
111
|
+
* so it also covers a URL that does not parse; the userinfo ends at the last `@` before the first
|
|
112
|
+
* whitespace, `/`, `?`, `#` or `'` after `://`. Linear in `text`, however many URLs it holds: each
|
|
113
|
+
* `://` is found with `indexOf`, its scheme by walking back no further than the previous `://`,
|
|
114
|
+
* and the search goes on right after each `://`, not after the authority it read: a URL glued
|
|
115
|
+
* onto the one before it (`https://host://u:p@h`, `https://u:p@x://v:q@w`) starts inside that
|
|
116
|
+
* authority, which ends at its `/`. An authority holds no `/`, so no `://` lies wholly inside one,
|
|
117
|
+
* and every character is read a bounded number of times: by the authority loop and `indexOf`
|
|
118
|
+
* forwards, by `schemeStart` backwards. The regex it replaces (a scheme tried at every
|
|
119
|
+
* word boundary) was quadratic: an 80 KB answer of `a.a.…` took about 5 s to print.
|
|
120
|
+
*/
|
|
121
|
+
export function redactUrlUserinfo(text) {
|
|
122
|
+
const parts = new TextBuilder();
|
|
123
|
+
let copied = 0;
|
|
124
|
+
let from = 0;
|
|
125
|
+
for (let at = text.indexOf("://"); at !== -1; at = text.indexOf("://", from)) {
|
|
126
|
+
const begin = at + 3;
|
|
127
|
+
let end = begin;
|
|
128
|
+
let lastAt = -1;
|
|
129
|
+
for (; end < text.length; end++) {
|
|
130
|
+
const c = text.charCodeAt(end);
|
|
131
|
+
if (c === 0x2f || c === 0x3f || c === 0x23 || c === 0x27 || /\s/.test(text[end]))
|
|
132
|
+
break;
|
|
133
|
+
if (c === 0x40)
|
|
134
|
+
lastAt = end;
|
|
135
|
+
}
|
|
136
|
+
if (lastAt !== -1 && schemeStart(text, at, from) !== -1) {
|
|
137
|
+
parts.push(text.slice(copied, begin), "***");
|
|
138
|
+
copied = lastAt;
|
|
139
|
+
}
|
|
140
|
+
from = begin;
|
|
141
|
+
}
|
|
142
|
+
if (copied === 0)
|
|
143
|
+
return text;
|
|
144
|
+
parts.push(text.slice(copied));
|
|
145
|
+
return parts.toString();
|
|
146
|
+
}
|
|
147
|
+
/** How many pieces a TextBuilder joins at a time. */
|
|
148
|
+
const BUILDER_BATCH = 16_384;
|
|
149
|
+
/**
|
|
150
|
+
* A text put together from many pieces (slices of a server answer and their replacements),
|
|
151
|
+
* joined in batches of BUILDER_BATCH into flat strings, so memory stays proportional to the text,
|
|
152
|
+
* not to the number of pieces. One `+=` per piece made a rope node each, and one array of every
|
|
153
|
+
* piece a slot each: millions of lone-surrogate repairs in a large answer exhausted a small heap
|
|
154
|
+
* either way.
|
|
155
|
+
*/
|
|
156
|
+
export class TextBuilder {
|
|
157
|
+
#done = [];
|
|
158
|
+
#batch = [];
|
|
159
|
+
push(...pieces) {
|
|
160
|
+
for (const piece of pieces)
|
|
161
|
+
this.#batch.push(piece);
|
|
162
|
+
if (this.#batch.length >= BUILDER_BATCH) {
|
|
163
|
+
this.#done.push(this.#batch.join(""));
|
|
164
|
+
this.#batch = [];
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
toString() {
|
|
168
|
+
this.#done.push(this.#batch.join(""));
|
|
169
|
+
this.#batch = [];
|
|
170
|
+
return this.#done.join("");
|
|
171
|
+
}
|
|
172
|
+
}
|
|
55
173
|
/**
|
|
56
174
|
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
-
* as `credentialsIn` returns it) back in an error body
|
|
58
|
-
* (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
59
|
-
* `user:password` itself, and the
|
|
60
|
-
*
|
|
61
|
-
*
|
|
175
|
+
* as `credentialsIn` returns it) back in an error body, in this order: the
|
|
176
|
+
* `Authorization: Basic` value (base64 of the decoded `user:password`, UTF-8 as the engine
|
|
177
|
+
* sends it), the decoded `user:password` itself, and the secret alone when it is at least 4
|
|
178
|
+
* characters long. The secret is the password; in a userinfo without one (`TOKEN@host`,
|
|
179
|
+
* `TOKEN:@host`) it is the user name, a token, which the engine sends as Basic `TOKEN:`. `[]`
|
|
180
|
+
* for an empty userinfo. None of them has an `@` to anchor on, so they are replaced as exact
|
|
181
|
+
* strings (`redactSecrets`).
|
|
62
182
|
*/
|
|
63
183
|
export function echoedCredentialForms(userinfo) {
|
|
64
184
|
const colon = userinfo.indexOf(":");
|
|
65
|
-
if (colon < 0)
|
|
66
|
-
return [];
|
|
67
185
|
const decode = (part) => {
|
|
68
186
|
try {
|
|
69
187
|
return decodeURIComponent(part);
|
|
@@ -72,14 +190,15 @@ export function echoedCredentialForms(userinfo) {
|
|
|
72
190
|
return part;
|
|
73
191
|
}
|
|
74
192
|
};
|
|
75
|
-
const user = decode(userinfo.slice(0, colon));
|
|
76
|
-
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
-
|
|
193
|
+
const user = decode(colon < 0 ? userinfo : userinfo.slice(0, colon));
|
|
194
|
+
const password = colon < 0 ? "" : decode(userinfo.slice(colon + 1));
|
|
195
|
+
const secret = password === "" ? user : password;
|
|
196
|
+
if (secret === "")
|
|
78
197
|
return [];
|
|
79
198
|
const pair = `${user}:${password}`;
|
|
80
199
|
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
-
if (
|
|
82
|
-
forms.push(
|
|
200
|
+
if (secret.length >= 4)
|
|
201
|
+
forms.push(secret);
|
|
83
202
|
return forms;
|
|
84
203
|
}
|
|
85
204
|
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
@@ -134,6 +253,35 @@ export function cutText(text, max) {
|
|
|
134
253
|
function isHighSurrogate(c) {
|
|
135
254
|
return c >= 0xd800 && c <= 0xdbff;
|
|
136
255
|
}
|
|
256
|
+
/** Unicode General Category Cf (format): bidi controls, zero-width characters, U+00AD, U+FEFF, … */
|
|
257
|
+
const FORMAT_CHAR = /\p{Cf}/u;
|
|
258
|
+
/**
|
|
259
|
+
* Quote a value for an error message, as `JSON.stringify` does, and also escape what it
|
|
260
|
+
* leaves raw although a terminal acts on it: DEL, the C1 controls, every Unicode
|
|
261
|
+
* format character (bidi overrides, zero-width characters) and the line and paragraph
|
|
262
|
+
* separators U+2028/U+2029, which a viewer may break the line at. A value is echoed in
|
|
263
|
+
* messages, and it can come from a pipeline or from the portal; an override in it would
|
|
264
|
+
* reorder the rest of the line ("Trojan Source"), a zero-width space would hide why it
|
|
265
|
+
* was refused. A value longer than MAX_MESSAGE_VALUE_LENGTH characters is cut, and the
|
|
266
|
+
* "…" goes after the closing quote (`"AAAA"…`), so it does not read as part of the value.
|
|
267
|
+
* The quoted part stays a valid JSON string.
|
|
268
|
+
*/
|
|
269
|
+
export function quoteValue(value) {
|
|
270
|
+
const cut = value.length > MAX_MESSAGE_VALUE_LENGTH;
|
|
271
|
+
let out = "";
|
|
272
|
+
for (const ch of JSON.stringify(cut ? cutText(value, MAX_MESSAGE_VALUE_LENGTH) : value)) {
|
|
273
|
+
const n = ch.codePointAt(0) ?? 0;
|
|
274
|
+
if ((n >= 0x7f && n <= 0x9f) || n === 0x2028 || n === 0x2029 || FORMAT_CHAR.test(ch)) {
|
|
275
|
+
// One escape per UTF-16 unit, so a character above U+FFFF stays valid JSON.
|
|
276
|
+
for (let i = 0; i < ch.length; i++)
|
|
277
|
+
out += `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`;
|
|
278
|
+
}
|
|
279
|
+
else {
|
|
280
|
+
out += ch;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
return cut ? `${out}…` : out;
|
|
284
|
+
}
|
|
137
285
|
/**
|
|
138
286
|
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
139
287
|
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
@@ -157,6 +305,12 @@ export class CkanApiError extends CkanError {
|
|
|
157
305
|
detail;
|
|
158
306
|
url;
|
|
159
307
|
method;
|
|
308
|
+
/**
|
|
309
|
+
* The answer's body as text, the base URL's credentials scrubbed. As sent, unless it is JSON
|
|
310
|
+
* whose decoded text still holds an echoed credential behind an escape: then the decoded JSON
|
|
311
|
+
* written again (other escapes, no insignificant whitespace) with it replaced by `***`, and `""`
|
|
312
|
+
* for such a body nested too deeply to write again.
|
|
313
|
+
*/
|
|
160
314
|
body;
|
|
161
315
|
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
162
316
|
retries;
|
|
@@ -201,17 +355,20 @@ export class CkanApiError extends CkanError {
|
|
|
201
355
|
/**
|
|
202
356
|
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
203
357
|
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
204
|
-
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
358
|
+
* message is `GET <url>: CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
205
359
|
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
206
360
|
* or undefined when the error carries none.
|
|
207
361
|
*/
|
|
208
362
|
export class CkanActionError extends CkanError {
|
|
209
363
|
action;
|
|
210
364
|
errorType;
|
|
365
|
+
/** The URL that answered, userinfo redacted; empty when the error was built without one. */
|
|
366
|
+
url;
|
|
211
367
|
constructor(message, args) {
|
|
212
368
|
super(message);
|
|
213
369
|
this.action = args.action;
|
|
214
370
|
this.errorType = args.errorType;
|
|
371
|
+
this.url = args.url === undefined ? "" : redactUrl(args.url);
|
|
215
372
|
}
|
|
216
373
|
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
217
374
|
get isNotFound() {
|
|
@@ -259,8 +416,21 @@ export function describeCkanError(error) {
|
|
|
259
416
|
* quotes and newlines escaped several levels deep.
|
|
260
417
|
*/
|
|
261
418
|
function solrReason(message) {
|
|
262
|
-
|
|
419
|
+
// Only a window after the first "[Reason: " is read: the pattern's lazy group, tried against
|
|
420
|
+
// `\\+n` at every step, is quadratic on a run of backslashes, and the message is uncut server
|
|
421
|
+
// text (80 KB took 2.4 s, past --timeout, and it grows with the square). The window is twice what a
|
|
422
|
+
// message shows; a reason that runs past it ends at the window's end.
|
|
423
|
+
const at = message.indexOf(REASON);
|
|
424
|
+
if (at === -1)
|
|
425
|
+
return undefined;
|
|
426
|
+
const end = at + REASON.length + 2 * MAX_MESSAGE_VALUE_LENGTH;
|
|
427
|
+
const window = message.slice(at, end);
|
|
428
|
+
const pattern = end < message.length ? SOLR_REASON_IN_CUT_WINDOW : SOLR_REASON;
|
|
429
|
+
const match = pattern.exec(window);
|
|
263
430
|
if (!match?.[1])
|
|
264
431
|
return undefined;
|
|
265
432
|
return match[1].replace(/\\+(['"])/g, "$1").trim();
|
|
266
433
|
}
|
|
434
|
+
const REASON = "[Reason: ";
|
|
435
|
+
const SOLR_REASON = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\])/;
|
|
436
|
+
const SOLR_REASON_IN_CUT_WINDOW = /^\[Reason: (?:[\w.$]+(?:Exception|Error): )?([\s\S]*?)(?:\\+n|\n|\]|$)/;
|
|
@@ -48,6 +48,7 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
|
|
|
48
48
|
/**
|
|
49
49
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
50
50
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
51
|
-
* (connection errors, timeouts, malformed URLs).
|
|
51
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
52
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
52
53
|
*/
|
|
53
54
|
export declare const nodeHttpTransport: Transport;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// `http.createServer` in the test-suite.
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
|
-
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
10
|
+
import { CkanNetworkError, redactUrl, credentialsIn, redactCredentials, typedUrlCredentials } from "./errors.js";
|
|
11
11
|
/** The message for a body over the size cap, naming the option on both sides. */
|
|
12
12
|
export function sizeLimitMessage(maxBytes) {
|
|
13
13
|
return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
|
|
@@ -17,10 +17,23 @@ export function sizeLimitMessage(maxBytes) {
|
|
|
17
17
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
18
18
|
*/
|
|
19
19
|
export const MAX_TIMEOUT_MS = 2_147_483_647;
|
|
20
|
+
/** The most received chunks the default transport keeps unjoined. */
|
|
21
|
+
const COALESCE_CHUNKS = 1024;
|
|
22
|
+
/** The bytes of unjoined chunks at which the default transport joins them into one Buffer. */
|
|
23
|
+
const COALESCE_BYTES = 64 * 1024;
|
|
24
|
+
/**
|
|
25
|
+
* `url` for a message, its userinfo redacted; one typed without its scheme (`user:pw@host`,
|
|
26
|
+
* `TOKEN:@host`) or with a mistyped one (`http//user:pw@host`) is read as a URL all the same
|
|
27
|
+
* (`typedUrlCredentials`), since `redactUrl` finds a userinfo only after a scheme.
|
|
28
|
+
*/
|
|
29
|
+
function shownUrl(url) {
|
|
30
|
+
return credentialsIn(url).length > 0 ? redactUrl(url) : redactCredentials(url, typedUrlCredentials(url));
|
|
31
|
+
}
|
|
20
32
|
/**
|
|
21
33
|
* Default transport. Resolves with the raw response (including non-2xx) — status
|
|
22
34
|
* interpretation is the client's job. Rejects only on transport-level failures
|
|
23
|
-
* (connection errors, timeouts, malformed URLs).
|
|
35
|
+
* (connection errors, timeouts, malformed URLs). The body's memory follows its bytes, not the
|
|
36
|
+
* number of chunks a server sends it in: small chunks are joined as they arrive.
|
|
24
37
|
*/
|
|
25
38
|
export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
26
39
|
let url;
|
|
@@ -28,14 +41,17 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
28
41
|
url = new URL(request.url);
|
|
29
42
|
}
|
|
30
43
|
catch {
|
|
31
|
-
reject(new CkanNetworkError(`Invalid URL: ${
|
|
44
|
+
reject(new CkanNetworkError(`Invalid URL: ${shownUrl(request.url)}`));
|
|
32
45
|
return;
|
|
33
46
|
}
|
|
34
47
|
// Only http/https are supported. Reject anything else up front with a clear,
|
|
35
48
|
// typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
|
|
36
49
|
// (and so this never reaches the file:/ftp:/etc. drivers).
|
|
37
50
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
38
|
-
|
|
51
|
+
// `user:password@host` without a scheme parses with the user name as the scheme: name a
|
|
52
|
+
// scheme only when the value spells one (`<scheme>://`).
|
|
53
|
+
const named = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(request.url) ? ` "${url.protocol}"` : "";
|
|
54
|
+
reject(new CkanNetworkError(`Unsupported protocol${named} in URL: ${shownUrl(request.url)}`));
|
|
39
55
|
return;
|
|
40
56
|
}
|
|
41
57
|
const isHttps = url.protocol === "https:";
|
|
@@ -62,7 +78,12 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
62
78
|
method: request.method,
|
|
63
79
|
headers: request.headers,
|
|
64
80
|
}, (res) => {
|
|
81
|
+
// Small chunks are joined as they arrive (`pending`, at most COALESCE_CHUNKS of them
|
|
82
|
+
// or COALESCE_BYTES), so the Buffers kept follow the bytes, not the number of chunks
|
|
83
|
+
// a server chooses to send: a Buffer per one-byte chunk held ~210 bytes of heap each.
|
|
65
84
|
const chunks = [];
|
|
85
|
+
let pending = [];
|
|
86
|
+
let pendingBytes = 0;
|
|
66
87
|
let received = 0;
|
|
67
88
|
let aborted = false;
|
|
68
89
|
res.on("data", (chunk) => {
|
|
@@ -75,7 +96,13 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
75
96
|
settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
|
|
76
97
|
return;
|
|
77
98
|
}
|
|
78
|
-
|
|
99
|
+
pending.push(chunk);
|
|
100
|
+
pendingBytes += chunk.length;
|
|
101
|
+
if (pending.length >= COALESCE_CHUNKS || pendingBytes >= COALESCE_BYTES) {
|
|
102
|
+
chunks.push(pending.length === 1 ? chunk : Buffer.concat(pending, pendingBytes));
|
|
103
|
+
pending = [];
|
|
104
|
+
pendingBytes = 0;
|
|
105
|
+
}
|
|
79
106
|
});
|
|
80
107
|
res.on("end", () => {
|
|
81
108
|
if (aborted)
|
|
@@ -83,7 +110,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
83
110
|
settleResolve({
|
|
84
111
|
status: res.statusCode ?? 0,
|
|
85
112
|
headers: res.headers,
|
|
86
|
-
body: Buffer.concat(chunks),
|
|
113
|
+
body: Buffer.concat([...chunks, ...pending]),
|
|
87
114
|
});
|
|
88
115
|
});
|
|
89
116
|
res.on("error", (err) => {
|
|
@@ -5,8 +5,8 @@ export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
-
export { assertValid, normalizeText } from "./validate.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
+
export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
|
12
12
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -3,8 +3,8 @@ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot }
|
|
|
3
3
|
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
-
export { assertValid, normalizeText } from "./validate.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, quoteValue, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
+
export { assertValid, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, sortProblem, textProblem, tokenProblem, } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
10
10
|
export * from "./types.js";
|
|
@@ -12,7 +12,10 @@ export declare function findPortal(idOrUrl: string, list: readonly Portal[]): Po
|
|
|
12
12
|
/** The outcome of `checkPortal`. */
|
|
13
13
|
export interface PortalCheck {
|
|
14
14
|
working: boolean;
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* A short reason when not working, e.g. `HTTP 404`, `host not found`, `not JSON (text/html)`,
|
|
17
|
+
* `invalid JSON`, `inexact number` (valid JSON with a number a double can't carry), `empty answer`.
|
|
18
|
+
*/
|
|
16
19
|
problem: string | null;
|
|
17
20
|
datasets: number | null;
|
|
18
21
|
/** From `status_show`, which some portals lock (Berlin): then null. */
|
|
@@ -94,13 +94,21 @@ function problemOf(err) {
|
|
|
94
94
|
}
|
|
95
95
|
const message = err instanceof Error ? err.message : String(err);
|
|
96
96
|
if (err instanceof CkanParseError) {
|
|
97
|
-
|
|
97
|
+
// The engine's "GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "<type>")."
|
|
98
|
+
const got = /\(expected JSON, got Content-Type "(.*)"\)\.$/.exec(message);
|
|
98
99
|
if (got)
|
|
99
100
|
return `not JSON (${got[1]})`;
|
|
100
|
-
if (
|
|
101
|
+
if (/: the answer is not a CKAN Action API response; /.test(message))
|
|
101
102
|
return "not a CKAN Action API";
|
|
102
|
-
if (
|
|
103
|
+
if (/: unexpected response shape: /.test(message))
|
|
103
104
|
return "unexpected response shape";
|
|
105
|
+
// Valid JSON whose number a double can't carry, or no body at all: not "invalid JSON".
|
|
106
|
+
if (/: the number .* cannot be represented exactly\.$/.test(message))
|
|
107
|
+
return "inexact number";
|
|
108
|
+
if (/: empty response body\.$/.test(message))
|
|
109
|
+
return "empty answer";
|
|
110
|
+
if (/: unsupported response charset /.test(message))
|
|
111
|
+
return "unsupported charset";
|
|
104
112
|
return "invalid JSON";
|
|
105
113
|
}
|
|
106
114
|
if (err instanceof CkanNetworkError) {
|