@maschinenlesbar.org/ckan-cli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +35 -8
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -6
- package/dist/src/cli/io.js +62 -10
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +72 -2
- package/dist/src/cli/run.d.ts +35 -10
- package/dist/src/cli/run.js +175 -25
- package/dist/src/cli/shared.d.ts +9 -1
- package/dist/src/cli/shared.js +17 -3
- package/dist/src/client/client.js +4 -3
- package/dist/src/client/engine.d.ts +15 -1
- package/dist/src/client/engine.js +51 -8
- package/dist/src/client/errors.d.ts +51 -7
- package/dist/src/client/errors.js +100 -16
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +2 -2
- package/dist/src/client/portals.js +3 -3
- package/dist/src/client/validate.js +2 -2
- package/package.json +4 -3
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { TextDecoder } from "node:util";
|
|
5
5
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
6
6
|
import { buildQueryString } from "./query.js";
|
|
7
|
-
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
|
|
7
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
8
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
9
9
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
10
10
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
@@ -116,6 +116,39 @@ export function sanitizeServerText(text) {
|
|
|
116
116
|
export function validateBaseUrl(raw) {
|
|
117
117
|
return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
118
118
|
}
|
|
119
|
+
/** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
|
|
120
|
+
function isLoopbackHost(hostname) {
|
|
121
|
+
return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
125
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
126
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
127
|
+
* where nothing leaves the machine.
|
|
128
|
+
*
|
|
129
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
130
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
131
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
132
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
133
|
+
* (once per run, before the first request).
|
|
134
|
+
*/
|
|
135
|
+
export function cleartextProblem(baseUrl, secrets = []) {
|
|
136
|
+
let url;
|
|
137
|
+
try {
|
|
138
|
+
url = new URL(baseUrl);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
|
|
144
|
+
return undefined;
|
|
145
|
+
const userinfo = url.username !== "" || url.password !== "";
|
|
146
|
+
const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
|
|
147
|
+
if (phrases.length === 0)
|
|
148
|
+
return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
|
|
149
|
+
const verb = phrases.length === 1 && !userinfo ? "is" : "are";
|
|
150
|
+
return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
|
|
151
|
+
}
|
|
119
152
|
// The headers the engine sets itself, under the exact keys it uses. They are the
|
|
120
153
|
// only ones that follow a cross-origin redirect.
|
|
121
154
|
const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
@@ -192,6 +225,12 @@ export class RequestEngine {
|
|
|
192
225
|
#baseUrl;
|
|
193
226
|
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
194
227
|
#credentials;
|
|
228
|
+
/**
|
|
229
|
+
* The forms a server echoes that userinfo back in (the Basic value, the decoded
|
|
230
|
+
* `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
|
|
231
|
+
* password never leaves half of the `user:password` around it.
|
|
232
|
+
*/
|
|
233
|
+
#echoed;
|
|
195
234
|
transport;
|
|
196
235
|
userAgent;
|
|
197
236
|
timeoutMs;
|
|
@@ -215,6 +254,9 @@ export class RequestEngine {
|
|
|
215
254
|
return [raw];
|
|
216
255
|
}
|
|
217
256
|
});
|
|
257
|
+
this.#echoed = credentialsIn(this.#baseUrl)
|
|
258
|
+
.flatMap(echoedCredentialForms)
|
|
259
|
+
.sort((a, b) => b.length - a.length);
|
|
218
260
|
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
219
261
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
220
262
|
this.userAgent =
|
|
@@ -230,13 +272,14 @@ export class RequestEngine {
|
|
|
230
272
|
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
231
273
|
}
|
|
232
274
|
/**
|
|
233
|
-
* `text` without the base URL's credentials (raw and percent-decoded
|
|
275
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
276
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
234
277
|
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
235
278
|
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
236
279
|
* messages built from server text.
|
|
237
280
|
*/
|
|
238
281
|
scrub(text) {
|
|
239
|
-
return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
|
|
282
|
+
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
240
283
|
}
|
|
241
284
|
/**
|
|
242
285
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
@@ -361,7 +404,7 @@ export class RequestEngine {
|
|
|
361
404
|
const finalUrl = response.url;
|
|
362
405
|
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
363
406
|
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
|
|
364
|
-
`(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
|
|
407
|
+
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
365
408
|
`(HttpRequest.redirect is "manual").`);
|
|
366
409
|
}
|
|
367
410
|
const status = response.status;
|
|
@@ -399,7 +442,7 @@ export class RequestEngine {
|
|
|
399
442
|
// Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
|
|
400
443
|
// never reaches a transport (a custom one may not check the scheme).
|
|
401
444
|
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
402
|
-
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
|
|
445
|
+
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
403
446
|
`for ${method} ${cutForMessage(redactUrl(url))}`);
|
|
404
447
|
}
|
|
405
448
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
@@ -435,7 +478,7 @@ export class RequestEngine {
|
|
|
435
478
|
const detail = target !== undefined
|
|
436
479
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
437
480
|
: location
|
|
438
|
-
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
481
|
+
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
439
482
|
: "redirect not followed (no Location header)";
|
|
440
483
|
throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
|
|
441
484
|
}
|
|
@@ -459,7 +502,7 @@ export class RequestEngine {
|
|
|
459
502
|
// path) typically answers with an HTML page and HTTP 200.
|
|
460
503
|
// The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
|
|
461
504
|
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
462
|
-
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
505
|
+
const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
|
|
463
506
|
const message = /json/i.test(mediaType)
|
|
464
507
|
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
465
508
|
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
@@ -550,7 +593,7 @@ function decodeBody(body, contentType, url) {
|
|
|
550
593
|
decoder = new TextDecoder(charset);
|
|
551
594
|
}
|
|
552
595
|
catch {
|
|
553
|
-
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
|
|
596
|
+
throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
|
|
554
597
|
}
|
|
555
598
|
return decoder.decode(body);
|
|
556
599
|
}
|
|
@@ -7,14 +7,26 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function redactUrl(url: string): string;
|
|
9
9
|
/**
|
|
10
|
-
* The userinfo a URL
|
|
11
|
-
* `https://alice:pa#ss@host` — or `[]` when it carries none.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
10
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
13
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
15
16
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
16
17
|
*/
|
|
17
18
|
export declare function credentialsIn(value: string): string[];
|
|
19
|
+
/**
|
|
20
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
22
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
23
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
24
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
25
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
26
|
+
*/
|
|
27
|
+
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
29
|
+
export declare function redactSecrets(text: string, secrets: readonly string[]): string;
|
|
18
30
|
/**
|
|
19
31
|
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
20
32
|
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
@@ -24,11 +36,26 @@ export declare function redactCredentials(text: string, credentials: readonly st
|
|
|
24
36
|
/**
|
|
25
37
|
* Longest URL or server text (in characters) an error message shows. A long search
|
|
26
38
|
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
27
|
-
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
39
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
40
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
41
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
42
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
28
43
|
*/
|
|
29
44
|
export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
30
|
-
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
45
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
31
46
|
export declare function cutForMessage(text: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
49
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
50
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
51
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
52
|
+
*/
|
|
53
|
+
export declare function cutText(text: string, max: number): string;
|
|
54
|
+
/**
|
|
55
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
56
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
57
|
+
*/
|
|
58
|
+
export declare function toWellFormed(text: string): string;
|
|
32
59
|
/** Base class for every error originating from this client. */
|
|
33
60
|
export declare class CkanError extends Error {
|
|
34
61
|
constructor(message: string, options?: {
|
|
@@ -66,6 +93,23 @@ export declare class CkanApiError extends CkanError {
|
|
|
66
93
|
/** True for statuses the API documents as transient and retry-able. */
|
|
67
94
|
get isRetryable(): boolean;
|
|
68
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
98
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
99
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
100
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
101
|
+
* or undefined when the error carries none.
|
|
102
|
+
*/
|
|
103
|
+
export declare class CkanActionError extends CkanError {
|
|
104
|
+
readonly action: string;
|
|
105
|
+
readonly errorType: string | undefined;
|
|
106
|
+
constructor(message: string, args: {
|
|
107
|
+
action: string;
|
|
108
|
+
errorType?: string | undefined;
|
|
109
|
+
});
|
|
110
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
111
|
+
get isNotFound(): boolean;
|
|
112
|
+
}
|
|
69
113
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
70
114
|
export declare class CkanNetworkError extends CkanError {
|
|
71
115
|
}
|
|
@@ -16,7 +16,8 @@ export function redactUrl(url) {
|
|
|
16
16
|
// A value that doesn't parse can still carry credentials: cut them out by text.
|
|
17
17
|
return redactCredentials(url, credentialsIn(url));
|
|
18
18
|
}
|
|
19
|
-
// `user:pw@host` without a scheme parses as a URL with
|
|
19
|
+
// A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
|
|
20
|
+
// the scheme "user:"), which is no URL with credentials at all.
|
|
20
21
|
if (parsed.username === "" && parsed.password === "")
|
|
21
22
|
return redactCredentials(url, credentialsIn(url));
|
|
22
23
|
parsed.username = "***";
|
|
@@ -24,24 +25,22 @@ export function redactUrl(url) {
|
|
|
24
25
|
return parsed.href;
|
|
25
26
|
}
|
|
26
27
|
/**
|
|
27
|
-
* The userinfo a URL
|
|
28
|
-
* `https://alice:pa#ss@host` — or `[]` when it carries none.
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
31
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
32
34
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
33
35
|
*/
|
|
34
36
|
export function credentialsIn(value) {
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
// Without a scheme only the unmistakable `user:password@host` form counts.
|
|
38
|
-
if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
|
|
37
|
+
const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
|
+
if (scheme === null)
|
|
39
39
|
return [];
|
|
40
|
-
|
|
41
|
-
const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
|
|
40
|
+
const rest = value.slice(scheme[0].length);
|
|
42
41
|
let parses = false;
|
|
43
42
|
try {
|
|
44
|
-
new URL(
|
|
43
|
+
new URL(value);
|
|
45
44
|
parses = true;
|
|
46
45
|
}
|
|
47
46
|
catch {
|
|
@@ -53,6 +52,46 @@ export function credentialsIn(value) {
|
|
|
53
52
|
const end = authority.lastIndexOf("@");
|
|
54
53
|
return end > 0 ? [rest.slice(0, end)] : [];
|
|
55
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
58
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
59
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
60
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
61
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
62
|
+
*/
|
|
63
|
+
export function echoedCredentialForms(userinfo) {
|
|
64
|
+
const colon = userinfo.indexOf(":");
|
|
65
|
+
if (colon < 0)
|
|
66
|
+
return [];
|
|
67
|
+
const decode = (part) => {
|
|
68
|
+
try {
|
|
69
|
+
return decodeURIComponent(part);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return part;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
const user = decode(userinfo.slice(0, colon));
|
|
76
|
+
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
+
if (password === "")
|
|
78
|
+
return [];
|
|
79
|
+
const pair = `${user}:${password}`;
|
|
80
|
+
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
+
if (password.length >= 4)
|
|
82
|
+
forms.push(password);
|
|
83
|
+
return forms;
|
|
84
|
+
}
|
|
85
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
86
|
+
export function redactSecrets(text, secrets) {
|
|
87
|
+
let out = text;
|
|
88
|
+
for (const secret of secrets) {
|
|
89
|
+
if (secret === "")
|
|
90
|
+
continue;
|
|
91
|
+
out = out.split(secret).join("***");
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
56
95
|
/**
|
|
57
96
|
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
58
97
|
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
@@ -70,12 +109,37 @@ export function redactCredentials(text, credentials) {
|
|
|
70
109
|
/**
|
|
71
110
|
* Longest URL or server text (in characters) an error message shows. A long search
|
|
72
111
|
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
73
|
-
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
112
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
113
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
114
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
115
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
74
116
|
*/
|
|
75
117
|
export const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
76
|
-
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
118
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
77
119
|
export function cutForMessage(text) {
|
|
78
|
-
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text
|
|
120
|
+
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
124
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
125
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
126
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
127
|
+
*/
|
|
128
|
+
export function cutText(text, max) {
|
|
129
|
+
if (text.length <= max)
|
|
130
|
+
return text;
|
|
131
|
+
const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
|
|
132
|
+
return text.slice(0, end);
|
|
133
|
+
}
|
|
134
|
+
function isHighSurrogate(c) {
|
|
135
|
+
return c >= 0xd800 && c <= 0xdbff;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
139
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
140
|
+
*/
|
|
141
|
+
export function toWellFormed(text) {
|
|
142
|
+
return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
|
|
79
143
|
}
|
|
80
144
|
/** Base class for every error originating from this client. */
|
|
81
145
|
export class CkanError extends Error {
|
|
@@ -134,6 +198,26 @@ export class CkanApiError extends CkanError {
|
|
|
134
198
|
return this.status === 429 || this.status === 503;
|
|
135
199
|
}
|
|
136
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
203
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
204
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
205
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
206
|
+
* or undefined when the error carries none.
|
|
207
|
+
*/
|
|
208
|
+
export class CkanActionError extends CkanError {
|
|
209
|
+
action;
|
|
210
|
+
errorType;
|
|
211
|
+
constructor(message, args) {
|
|
212
|
+
super(message);
|
|
213
|
+
this.action = args.action;
|
|
214
|
+
this.errorType = args.errorType;
|
|
215
|
+
}
|
|
216
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
217
|
+
get isNotFound() {
|
|
218
|
+
return this.errorType === "Not Found Error";
|
|
219
|
+
}
|
|
220
|
+
}
|
|
137
221
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
138
222
|
export class CkanNetworkError extends CkanError {
|
|
139
223
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
2
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
3
3
|
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
9
|
export { assertValid } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
package/dist/src/client/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
2
|
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
3
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
3
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
7
|
export { assertValid } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// portals-list.ts, maintained with scripts/update-portals.ts.
|
|
3
3
|
import { CkanClient, siteRoot } from "./client.js";
|
|
4
4
|
import { validateBaseUrl } from "./engine.js";
|
|
5
|
-
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
|
|
5
|
+
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
|
|
6
6
|
import { PORTALS } from "./portals-list.js";
|
|
7
7
|
import { assertValid, intRangeProblem } from "./validate.js";
|
|
8
8
|
/**
|
|
@@ -118,9 +118,9 @@ function problemOf(err) {
|
|
|
118
118
|
// The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
|
|
119
119
|
const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
|
|
120
120
|
if (reason !== undefined)
|
|
121
|
-
return reason
|
|
121
|
+
return cutText(reason, 80);
|
|
122
122
|
}
|
|
123
|
-
return message
|
|
123
|
+
return cutText(message, 80);
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
126
|
* A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the reason a value is invalid, or `undefined` when it is valid. Client methods
|
|
3
3
|
// enforce them with `assertValid` before any request; the CLI's value parsers call
|
|
4
4
|
// the same functions, so a rule is written once and the CLI and the library agree.
|
|
5
|
-
import { CkanValidationError } from "./errors.js";
|
|
5
|
+
import { CkanValidationError, cutForMessage } from "./errors.js";
|
|
6
6
|
/**
|
|
7
7
|
* Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
|
|
8
8
|
* something wrong with `value`; otherwise return `value` unchanged. A method that
|
|
@@ -147,7 +147,7 @@ export function baseUrlProblem(value) {
|
|
|
147
147
|
return "Expected an absolute http(s) URL.";
|
|
148
148
|
}
|
|
149
149
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
150
|
-
return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
|
|
150
|
+
return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
|
|
151
151
|
}
|
|
152
152
|
if (/[?#]/.test(value))
|
|
153
153
|
return "Expected a site URL without a query string or fragment.";
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@maschinenlesbar.org/ckan-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -18,7 +18,8 @@
|
|
|
18
18
|
"dist/src",
|
|
19
19
|
"!dist/src/**/*.map",
|
|
20
20
|
"LICENSING.md",
|
|
21
|
-
"CONTRIBUTING.md"
|
|
21
|
+
"CONTRIBUTING.md",
|
|
22
|
+
"DATA_LICENSE.md"
|
|
22
23
|
],
|
|
23
24
|
"engines": {
|
|
24
25
|
"node": ">=22.12"
|
|
@@ -29,7 +30,7 @@
|
|
|
29
30
|
"prepack": "npm run build",
|
|
30
31
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
31
32
|
"pretest": "npm run build",
|
|
32
|
-
"test": "node --test dist/test/*.test.js",
|
|
33
|
+
"test": "node --test --test-timeout=5000 dist/test/*.test.js",
|
|
33
34
|
"docs": "node tools/docs/node_modules/typedoc/bin/typedoc src/index.ts --out out",
|
|
34
35
|
"start": "node dist/src/cli/index.js",
|
|
35
36
|
"update-portals": "npm run build && node dist/scripts/update-portals.js"
|