@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +55 -12
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -5
- package/dist/src/cli/io.js +65 -9
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +88 -11
- package/dist/src/cli/run.d.ts +53 -1
- package/dist/src/cli/run.js +239 -8
- package/dist/src/cli/shared.d.ts +17 -1
- package/dist/src/cli/shared.js +33 -3
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +124 -22
- package/dist/src/client/engine.d.ts +62 -9
- package/dist/src/client/engine.js +350 -47
- package/dist/src/client/errors.d.ts +81 -1
- package/dist/src/client/errors.js +171 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +3 -3
- package/dist/src/client/index.js +3 -3
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +39 -4
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +49 -3
- package/package.json +6 -5
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
5
5
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
6
|
-
* A
|
|
6
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
7
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
8
|
+
* value without userinfo is returned unchanged.
|
|
7
9
|
*/
|
|
8
10
|
export function redactUrl(url) {
|
|
9
11
|
let parsed;
|
|
@@ -11,14 +13,134 @@ export function redactUrl(url) {
|
|
|
11
13
|
parsed = new URL(url);
|
|
12
14
|
}
|
|
13
15
|
catch {
|
|
14
|
-
|
|
16
|
+
// A value that doesn't parse can still carry credentials: cut them out by text.
|
|
17
|
+
return redactCredentials(url, credentialsIn(url));
|
|
15
18
|
}
|
|
19
|
+
// A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
|
|
20
|
+
// the scheme "user:"), which is no URL with credentials at all.
|
|
16
21
|
if (parsed.username === "" && parsed.password === "")
|
|
17
|
-
return url;
|
|
22
|
+
return redactCredentials(url, credentialsIn(url));
|
|
18
23
|
parsed.username = "***";
|
|
19
24
|
parsed.password = "";
|
|
20
25
|
return parsed.href;
|
|
21
26
|
}
|
|
27
|
+
/**
|
|
28
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
31
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
34
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
35
|
+
*/
|
|
36
|
+
export function credentialsIn(value) {
|
|
37
|
+
const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
|
+
if (scheme === null)
|
|
39
|
+
return [];
|
|
40
|
+
const rest = value.slice(scheme[0].length);
|
|
41
|
+
let parses = false;
|
|
42
|
+
try {
|
|
43
|
+
new URL(value);
|
|
44
|
+
parses = true;
|
|
45
|
+
}
|
|
46
|
+
catch {
|
|
47
|
+
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
48
|
+
}
|
|
49
|
+
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
50
|
+
// first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
|
|
51
|
+
const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
|
|
52
|
+
const end = authority.lastIndexOf("@");
|
|
53
|
+
return end > 0 ? [rest.slice(0, end)] : [];
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
58
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
59
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
60
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
61
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
62
|
+
*/
|
|
63
|
+
export function echoedCredentialForms(userinfo) {
|
|
64
|
+
const colon = userinfo.indexOf(":");
|
|
65
|
+
if (colon < 0)
|
|
66
|
+
return [];
|
|
67
|
+
const decode = (part) => {
|
|
68
|
+
try {
|
|
69
|
+
return decodeURIComponent(part);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return part;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
const user = decode(userinfo.slice(0, colon));
|
|
76
|
+
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
+
if (password === "")
|
|
78
|
+
return [];
|
|
79
|
+
const pair = `${user}:${password}`;
|
|
80
|
+
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
+
if (password.length >= 4)
|
|
82
|
+
forms.push(password);
|
|
83
|
+
return forms;
|
|
84
|
+
}
|
|
85
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
86
|
+
export function redactSecrets(text, secrets) {
|
|
87
|
+
let out = text;
|
|
88
|
+
for (const secret of secrets) {
|
|
89
|
+
if (secret === "")
|
|
90
|
+
continue;
|
|
91
|
+
out = out.split(secret).join("***");
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
97
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
98
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
99
|
+
*/
|
|
100
|
+
export function redactCredentials(text, credentials) {
|
|
101
|
+
let out = text;
|
|
102
|
+
for (const secret of credentials) {
|
|
103
|
+
if (secret === "")
|
|
104
|
+
continue;
|
|
105
|
+
out = out.split(`${secret}@`).join("***@");
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
111
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
112
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
113
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
114
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
115
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
116
|
+
*/
|
|
117
|
+
export const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
118
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
119
|
+
export function cutForMessage(text) {
|
|
120
|
+
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
124
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
125
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
126
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
127
|
+
*/
|
|
128
|
+
export function cutText(text, max) {
|
|
129
|
+
if (text.length <= max)
|
|
130
|
+
return text;
|
|
131
|
+
const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
|
|
132
|
+
return text.slice(0, end);
|
|
133
|
+
}
|
|
134
|
+
function isHighSurrogate(c) {
|
|
135
|
+
return c >= 0xd800 && c <= 0xdbff;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
139
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
140
|
+
*/
|
|
141
|
+
export function toWellFormed(text) {
|
|
142
|
+
return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
|
|
143
|
+
}
|
|
22
144
|
/** Base class for every error originating from this client. */
|
|
23
145
|
export class CkanError extends Error {
|
|
24
146
|
constructor(message, options) {
|
|
@@ -36,22 +158,66 @@ export class CkanApiError extends CkanError {
|
|
|
36
158
|
url;
|
|
37
159
|
method;
|
|
38
160
|
body;
|
|
161
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
162
|
+
retries;
|
|
163
|
+
/**
|
|
164
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
165
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
166
|
+
* else undefined.
|
|
167
|
+
*/
|
|
168
|
+
retryAfterMs;
|
|
39
169
|
constructor(args) {
|
|
40
170
|
// The URL is shown without userinfo: a credential in --base-url must not leak.
|
|
41
171
|
const url = redactUrl(args.url);
|
|
42
|
-
const
|
|
43
|
-
|
|
172
|
+
const parts = [];
|
|
173
|
+
if (args.detail)
|
|
174
|
+
parts.push(cutForMessage(args.detail));
|
|
175
|
+
if (args.retryAfterMs !== undefined) {
|
|
176
|
+
// Say why the retries the caller asked for never ran: the server asked for a
|
|
177
|
+
// wait longer than the engine sleeps, and retrying earlier would land inside it.
|
|
178
|
+
const wait = Math.ceil(args.retryAfterMs / 1000);
|
|
179
|
+
const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
|
|
180
|
+
parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
|
|
181
|
+
}
|
|
182
|
+
const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
|
|
183
|
+
const retries = args.retries ?? 0;
|
|
184
|
+
// Say that the status persisted through retries, so a user knows whether raising
|
|
185
|
+
// --max-retries could help.
|
|
186
|
+
const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
|
|
187
|
+
super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
|
|
44
188
|
this.status = args.status;
|
|
45
189
|
this.url = url;
|
|
46
190
|
this.method = args.method;
|
|
47
191
|
this.body = args.body;
|
|
48
192
|
this.detail = args.detail;
|
|
193
|
+
this.retries = retries;
|
|
194
|
+
this.retryAfterMs = args.retryAfterMs;
|
|
49
195
|
}
|
|
50
196
|
/** True for statuses the API documents as transient and retry-able. */
|
|
51
197
|
get isRetryable() {
|
|
52
198
|
return this.status === 429 || this.status === 503;
|
|
53
199
|
}
|
|
54
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
203
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
204
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
205
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
206
|
+
* or undefined when the error carries none.
|
|
207
|
+
*/
|
|
208
|
+
export class CkanActionError extends CkanError {
|
|
209
|
+
action;
|
|
210
|
+
errorType;
|
|
211
|
+
constructor(message, args) {
|
|
212
|
+
super(message);
|
|
213
|
+
this.action = args.action;
|
|
214
|
+
this.errorType = args.errorType;
|
|
215
|
+
}
|
|
216
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
217
|
+
get isNotFound() {
|
|
218
|
+
return this.errorType === "Not Found Error";
|
|
219
|
+
}
|
|
220
|
+
}
|
|
55
221
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
56
222
|
export class CkanNetworkError extends CkanError {
|
|
57
223
|
}
|
|
@@ -10,13 +10,36 @@ export interface HttpRequest {
|
|
|
10
10
|
timeoutMs?: number;
|
|
11
11
|
/** Hard cap on the response body size in bytes; the request aborts if exceeded. */
|
|
12
12
|
maxResponseBytes?: number;
|
|
13
|
+
/**
|
|
14
|
+
* Aborted when the engine's overall deadline (`timeoutMs`) passes. A transport should
|
|
15
|
+
* stop the request then (`fetch(url, { signal })`); the engine rejects at the deadline
|
|
16
|
+
* either way, and enforces `maxResponseBytes` on the body it gets back, so neither
|
|
17
|
+
* limit depends on the transport.
|
|
18
|
+
*/
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
/**
|
|
21
|
+
* Always `"manual"` from the engine: a transport must not follow redirects. The
|
|
22
|
+
* engine follows them itself and decides per hop whether the `Authorization`
|
|
23
|
+
* header (the base URL's userinfo) goes along (same origin only). A fetch-based
|
|
24
|
+
* transport passes it on: `fetch(url, { redirect })`.
|
|
25
|
+
*/
|
|
26
|
+
redirect?: "manual";
|
|
13
27
|
}
|
|
14
28
|
export interface HttpResponse {
|
|
15
29
|
status: number;
|
|
16
30
|
headers: http.IncomingHttpHeaders;
|
|
17
31
|
body: Buffer;
|
|
32
|
+
/**
|
|
33
|
+
* The URL the response came from, if the transport knows it (fetch's
|
|
34
|
+
* `response.url`). When it is on another origin than the request, the transport
|
|
35
|
+
* followed a redirect itself and the engine rejects the response with a
|
|
36
|
+
* CkanNetworkError.
|
|
37
|
+
*/
|
|
38
|
+
url?: string;
|
|
18
39
|
}
|
|
19
40
|
export type Transport = (request: HttpRequest) => Promise<HttpResponse>;
|
|
41
|
+
/** The message for a body over the size cap, naming the option on both sides. */
|
|
42
|
+
export declare function sizeLimitMessage(maxBytes: number): string;
|
|
20
43
|
/**
|
|
21
44
|
* The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
|
|
22
45
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
package/dist/src/client/http.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
10
|
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
11
|
+
/** The message for a body over the size cap, naming the option on both sides. */
|
|
12
|
+
export function sizeLimitMessage(maxBytes) {
|
|
13
|
+
return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
|
|
14
|
+
}
|
|
11
15
|
/**
|
|
12
16
|
* The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
|
|
13
17
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
@@ -68,7 +72,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
68
72
|
if (maxBytes !== undefined && received > maxBytes) {
|
|
69
73
|
aborted = true;
|
|
70
74
|
res.destroy();
|
|
71
|
-
settleReject(new CkanNetworkError(
|
|
75
|
+
settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
|
|
72
76
|
return;
|
|
73
77
|
}
|
|
74
78
|
chunks.push(chunk);
|
|
@@ -109,6 +113,15 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
109
113
|
// Don't let a pending deadline timer keep the event loop alive on its own.
|
|
110
114
|
deadline.unref?.();
|
|
111
115
|
}
|
|
116
|
+
if (request.signal !== undefined) {
|
|
117
|
+
const abort = () => {
|
|
118
|
+
req.destroy(new CkanNetworkError(`Request exceeded the ${request.timeoutMs ?? 0}ms deadline`));
|
|
119
|
+
};
|
|
120
|
+
if (request.signal.aborted)
|
|
121
|
+
abort();
|
|
122
|
+
else
|
|
123
|
+
request.signal.addEventListener("abort", abort, { once: true });
|
|
124
|
+
}
|
|
112
125
|
req.on("error", (err) => {
|
|
113
126
|
// A timeout destroy already passes an CkanNetworkError; don't double-wrap.
|
|
114
127
|
settleReject(err instanceof CkanNetworkError ? err : new CkanNetworkError(err.message, { cause: err }));
|
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
export { CkanClient, siteRoot } from "./client.js";
|
|
2
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
1
|
+
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
3
3
|
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
9
|
export { assertValid } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
package/dist/src/client/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
|
-
export { CkanClient, siteRoot } from "./client.js";
|
|
3
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
2
|
+
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
3
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
7
|
export { assertValid } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { CkanClient } from "./client.js";
|
|
2
|
-
import type
|
|
2
|
+
import { type EngineOptions } from "./engine.js";
|
|
3
3
|
import type { Portal } from "./types.js";
|
|
4
4
|
/**
|
|
5
5
|
* One identity per portal, whatever form its URL was written in: host (without
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// Known CKAN portals: lookup and a short live check. The list itself is in
|
|
2
2
|
// portals-list.ts, maintained with scripts/update-portals.ts.
|
|
3
3
|
import { CkanClient, siteRoot } from "./client.js";
|
|
4
|
-
import {
|
|
4
|
+
import { validateBaseUrl } from "./engine.js";
|
|
5
|
+
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
|
|
5
6
|
import { PORTALS } from "./portals-list.js";
|
|
6
7
|
import { assertValid, intRangeProblem } from "./validate.js";
|
|
7
8
|
/**
|
|
@@ -17,6 +18,10 @@ export function portalKey(url) {
|
|
|
17
18
|
}
|
|
18
19
|
/** A portal from `list` by its id (any case) or by any form of its URL. */
|
|
19
20
|
export function findPortal(idOrUrl, list) {
|
|
21
|
+
if (typeof idOrUrl !== "string")
|
|
22
|
+
throw new CkanValidationError("Invalid portal: Expected a string.");
|
|
23
|
+
if (!Array.isArray(list))
|
|
24
|
+
throw new CkanValidationError("Invalid portal list: Expected an array.");
|
|
20
25
|
const wanted = idOrUrl.trim().toLowerCase();
|
|
21
26
|
const byId = list.find((p) => p.id.toLowerCase() === wanted);
|
|
22
27
|
if (byId)
|
|
@@ -28,7 +33,14 @@ export function findPortal(idOrUrl, list) {
|
|
|
28
33
|
catch {
|
|
29
34
|
return undefined; // not a URL either
|
|
30
35
|
}
|
|
31
|
-
return list.find((p) =>
|
|
36
|
+
return list.find((p) => {
|
|
37
|
+
try {
|
|
38
|
+
return portalKey(p.url) === key;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false; // an entry without a usable url matches nothing
|
|
42
|
+
}
|
|
43
|
+
});
|
|
32
44
|
}
|
|
33
45
|
/**
|
|
34
46
|
* A short live check of the CKAN portal a client points at. It works when `package_search?rows=0`
|
|
@@ -64,6 +76,17 @@ export async function checkPortal(client) {
|
|
|
64
76
|
}
|
|
65
77
|
return result;
|
|
66
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* The first error `code` (`ENOTFOUND`, `ECONNREFUSED`, …) along a cause chain: the
|
|
81
|
+
* engine wraps a transport failure in a CkanNetworkError that names the request, so
|
|
82
|
+
* Node's code sits one or more levels down.
|
|
83
|
+
*/
|
|
84
|
+
function errorCode(cause, depth = 0) {
|
|
85
|
+
if (typeof cause !== "object" || cause === null || depth > 5)
|
|
86
|
+
return undefined;
|
|
87
|
+
const code = cause.code;
|
|
88
|
+
return code !== undefined ? code : errorCode(cause.cause, depth + 1);
|
|
89
|
+
}
|
|
67
90
|
/** A short, stable reason for a failed check. */
|
|
68
91
|
function problemOf(err) {
|
|
69
92
|
if (err instanceof CkanApiError) {
|
|
@@ -81,7 +104,7 @@ function problemOf(err) {
|
|
|
81
104
|
return "invalid JSON";
|
|
82
105
|
}
|
|
83
106
|
if (err instanceof CkanNetworkError) {
|
|
84
|
-
const code = err.cause
|
|
107
|
+
const code = errorCode(err.cause);
|
|
85
108
|
if (code === "ENOTFOUND" || code === "EAI_AGAIN")
|
|
86
109
|
return "host not found";
|
|
87
110
|
if (code === "ECONNREFUSED")
|
|
@@ -92,8 +115,12 @@ function problemOf(err) {
|
|
|
92
115
|
return "TLS certificate not verifiable";
|
|
93
116
|
if (/timed out|deadline/.test(message))
|
|
94
117
|
return "timeout";
|
|
118
|
+
// The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
|
|
119
|
+
const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
|
|
120
|
+
if (reason !== undefined)
|
|
121
|
+
return cutText(reason, 80);
|
|
95
122
|
}
|
|
96
|
-
return message
|
|
123
|
+
return cutText(message, 80);
|
|
97
124
|
}
|
|
98
125
|
/**
|
|
99
126
|
* A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
|
|
@@ -133,6 +160,8 @@ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
|
133
160
|
* CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
|
|
134
161
|
*/
|
|
135
162
|
export async function checkPortalUrls(urls, options = {}) {
|
|
163
|
+
if (!Array.isArray(urls))
|
|
164
|
+
throw new CkanValidationError("Invalid urls: Expected an array.");
|
|
136
165
|
const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
|
|
137
166
|
const { retryDelayMs } = options;
|
|
138
167
|
if (retryDelayMs !== undefined)
|
|
@@ -142,6 +171,10 @@ export async function checkPortalUrls(urls, options = {}) {
|
|
|
142
171
|
return mapLimit(urls, concurrency, async (url) => {
|
|
143
172
|
let client;
|
|
144
173
|
try {
|
|
174
|
+
// Checked here, not left to the client: `baseUrl: undefined` selects the
|
|
175
|
+
// Hamburg default, so an entry without a url was checked against Hamburg and
|
|
176
|
+
// reported as working, with Hamburg's dataset count.
|
|
177
|
+
validateBaseUrl(url);
|
|
145
178
|
client = createClient({ ...options.engineOptions, baseUrl: url });
|
|
146
179
|
}
|
|
147
180
|
catch (err) {
|
|
@@ -160,6 +193,8 @@ export async function checkPortalUrls(urls, options = {}) {
|
|
|
160
193
|
* What `ckan portals --check` prints.
|
|
161
194
|
*/
|
|
162
195
|
export async function checkPortals(portals = PORTALS, options = {}) {
|
|
196
|
+
if (!Array.isArray(portals))
|
|
197
|
+
throw new CkanValidationError("Invalid portals: Expected an array.");
|
|
163
198
|
const date = options.date ?? new Date().toISOString().slice(0, 10);
|
|
164
199
|
const checks = await checkPortalUrls(portals.map((p) => p.url), options);
|
|
165
200
|
return portals.map((portal, i) => withCheck(portal, checks[i], date));
|
|
@@ -15,6 +15,16 @@ export declare function isBlank(value: string): boolean;
|
|
|
15
15
|
* silently return the unfiltered result. Values that are not strings pass.
|
|
16
16
|
*/
|
|
17
17
|
export declare function blankProblem(value: unknown): string | undefined;
|
|
18
|
+
/** A non-blank string (an id, a query, a sort expression); anything else is refused. */
|
|
19
|
+
export declare function textProblem(value: unknown): string | undefined;
|
|
20
|
+
/** A list of non-blank strings (filter queries, facet fields). */
|
|
21
|
+
export declare function textListProblem(value: unknown): string | undefined;
|
|
22
|
+
/**
|
|
23
|
+
* A value the query-string builder can send: a string, a finite number, a boolean
|
|
24
|
+
* or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
|
|
25
|
+
* function or `NaN` would go out as `[object Object]` or `NaN`.
|
|
26
|
+
*/
|
|
27
|
+
export declare function queryValueProblem(value: unknown): string | undefined;
|
|
18
28
|
/**
|
|
19
29
|
* A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
|
|
20
30
|
* A negative, fractional or non-finite value would go upstream as given, and
|
|
@@ -47,7 +57,9 @@ export declare function headerValueProblem(value: string): string | undefined;
|
|
|
47
57
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
48
58
|
* transport);
|
|
49
59
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
50
|
-
* front of `/api/3/action
|
|
60
|
+
* front of `/api/3/action`;
|
|
61
|
+
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
62
|
+
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
51
63
|
*
|
|
52
64
|
* Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
|
|
53
65
|
* reasons never quote the value, so a credential in it cannot leak through them.
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the reason a value is invalid, or `undefined` when it is valid. Client methods
|
|
3
3
|
// enforce them with `assertValid` before any request; the CLI's value parsers call
|
|
4
4
|
// the same functions, so a rule is written once and the CLI and the library agree.
|
|
5
|
-
import { CkanValidationError } from "./errors.js";
|
|
5
|
+
import { CkanValidationError, cutForMessage } from "./errors.js";
|
|
6
6
|
/**
|
|
7
7
|
* Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
|
|
8
8
|
* something wrong with `value`; otherwise return `value` unchanged. A method that
|
|
@@ -30,6 +30,36 @@ export function blankProblem(value) {
|
|
|
30
30
|
return "Expected a non-empty value.";
|
|
31
31
|
return undefined;
|
|
32
32
|
}
|
|
33
|
+
/** A non-blank string (an id, a query, a sort expression); anything else is refused. */
|
|
34
|
+
export function textProblem(value) {
|
|
35
|
+
if (typeof value !== "string")
|
|
36
|
+
return "Expected a string.";
|
|
37
|
+
return blankProblem(value);
|
|
38
|
+
}
|
|
39
|
+
/** A list of non-blank strings (filter queries, facet fields). */
|
|
40
|
+
export function textListProblem(value) {
|
|
41
|
+
if (!Array.isArray(value))
|
|
42
|
+
return "Expected a list of strings.";
|
|
43
|
+
if (value.some((v) => typeof v !== "string"))
|
|
44
|
+
return "Expected a list of strings.";
|
|
45
|
+
return blankProblem(value);
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* A value the query-string builder can send: a string, a finite number, a boolean
|
|
49
|
+
* or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
|
|
50
|
+
* function or `NaN` would go out as `[object Object]` or `NaN`.
|
|
51
|
+
*/
|
|
52
|
+
export function queryValueProblem(value) {
|
|
53
|
+
const ok = (v) => v === null ||
|
|
54
|
+
v === undefined ||
|
|
55
|
+
typeof v === "string" ||
|
|
56
|
+
typeof v === "boolean" ||
|
|
57
|
+
(typeof v === "number" && Number.isFinite(v)) ||
|
|
58
|
+
(v instanceof Date && !Number.isNaN(v.getTime()));
|
|
59
|
+
if (Array.isArray(value) ? value.every(ok) : ok(value))
|
|
60
|
+
return blankProblem(value);
|
|
61
|
+
return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
|
|
62
|
+
}
|
|
33
63
|
/**
|
|
34
64
|
* A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
|
|
35
65
|
* A negative, fractional or non-finite value would go upstream as given, and
|
|
@@ -65,6 +95,8 @@ export function intRangeProblem(min, max) {
|
|
|
65
95
|
* so the source stays free of control bytes.
|
|
66
96
|
*/
|
|
67
97
|
export function headerValueProblem(value) {
|
|
98
|
+
if (typeof value !== "string")
|
|
99
|
+
return "Expected a string.";
|
|
68
100
|
const blank = blankProblem(value);
|
|
69
101
|
if (blank !== undefined)
|
|
70
102
|
return blank;
|
|
@@ -88,12 +120,16 @@ export function headerValueProblem(value) {
|
|
|
88
120
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
89
121
|
* transport);
|
|
90
122
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
91
|
-
* front of `/api/3/action
|
|
123
|
+
* front of `/api/3/action`;
|
|
124
|
+
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
125
|
+
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
92
126
|
*
|
|
93
127
|
* Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
|
|
94
128
|
* reasons never quote the value, so a credential in it cannot leak through them.
|
|
95
129
|
*/
|
|
96
130
|
export function baseUrlProblem(value) {
|
|
131
|
+
if (typeof value !== "string")
|
|
132
|
+
return "Expected a string with an absolute http(s) URL.";
|
|
97
133
|
if (isBlank(value))
|
|
98
134
|
return "Expected an absolute http(s) URL.";
|
|
99
135
|
if (value !== value.trim())
|
|
@@ -111,9 +147,19 @@ export function baseUrlProblem(value) {
|
|
|
111
147
|
return "Expected an absolute http(s) URL.";
|
|
112
148
|
}
|
|
113
149
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
114
|
-
return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
|
|
150
|
+
return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
|
|
115
151
|
}
|
|
116
152
|
if (/[?#]/.test(value))
|
|
117
153
|
return "Expected a site URL without a query string or fragment.";
|
|
154
|
+
// The engine decodes the userinfo into the Authorization header; a "%" that isn't an
|
|
155
|
+
// escape would fail there ("URI malformed") at request time. Reject it here.
|
|
156
|
+
for (const part of [url.username, url.password]) {
|
|
157
|
+
try {
|
|
158
|
+
decodeURIComponent(part);
|
|
159
|
+
}
|
|
160
|
+
catch {
|
|
161
|
+
return 'The user name or password has a "%" that is not followed by two hex digits; write a literal "%" as %25.';
|
|
162
|
+
}
|
|
163
|
+
}
|
|
118
164
|
return undefined;
|
|
119
165
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@maschinenlesbar.org/ckan-cli",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -18,10 +18,11 @@
|
|
|
18
18
|
"dist/src",
|
|
19
19
|
"!dist/src/**/*.map",
|
|
20
20
|
"LICENSING.md",
|
|
21
|
-
"CONTRIBUTING.md"
|
|
21
|
+
"CONTRIBUTING.md",
|
|
22
|
+
"DATA_LICENSE.md"
|
|
22
23
|
],
|
|
23
24
|
"engines": {
|
|
24
|
-
"node": ">=
|
|
25
|
+
"node": ">=22.12"
|
|
25
26
|
},
|
|
26
27
|
"scripts": {
|
|
27
28
|
"clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
|
|
@@ -29,7 +30,7 @@
|
|
|
29
30
|
"prepack": "npm run build",
|
|
30
31
|
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
31
32
|
"pretest": "npm run build",
|
|
32
|
-
"test": "node --test dist/test/*.test.js",
|
|
33
|
+
"test": "node --test --test-timeout=5000 dist/test/*.test.js",
|
|
33
34
|
"docs": "node tools/docs/node_modules/typedoc/bin/typedoc src/index.ts --out out",
|
|
34
35
|
"start": "node dist/src/cli/index.js",
|
|
35
36
|
"update-portals": "npm run build && node dist/scripts/update-portals.js"
|
|
@@ -66,7 +67,7 @@
|
|
|
66
67
|
"commander": "15.0.0"
|
|
67
68
|
},
|
|
68
69
|
"devDependencies": {
|
|
69
|
-
"@types/node": "^
|
|
70
|
+
"@types/node": "^22.20.5",
|
|
70
71
|
"typescript": "7.0.2"
|
|
71
72
|
}
|
|
72
73
|
}
|