@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +23 -15
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +13 -12
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +0 -1
- package/dist/src/cli/io.d.ts +2 -2
- package/dist/src/cli/io.js +6 -3
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +17 -11
- package/dist/src/cli/run.d.ts +27 -1
- package/dist/src/cli/run.js +90 -2
- package/dist/src/cli/shared.d.ts +18 -15
- package/dist/src/cli/shared.js +45 -54
- package/dist/src/client/client.d.ts +30 -9
- package/dist/src/client/client.js +154 -37
- package/dist/src/client/engine.d.ts +74 -13
- package/dist/src/client/engine.js +341 -71
- package/dist/src/client/errors.d.ts +44 -2
- package/dist/src/client/errors.js +94 -6
- package/dist/src/client/http.d.ts +23 -1
- package/dist/src/client/http.js +54 -33
- package/dist/src/client/index.d.ts +7 -6
- package/dist/src/client/index.js +5 -5
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +86 -6
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +0 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +67 -0
- package/dist/src/client/validate.js +165 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +4 -3
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
|
@@ -1,9 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
3
3
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
4
|
-
* A
|
|
4
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
5
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
6
|
+
* value without userinfo is returned unchanged.
|
|
5
7
|
*/
|
|
6
8
|
export declare function redactUrl(url: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
|
|
12
|
+
* parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
|
|
13
|
+
* is everything between `://` and the last `@` before the host. A value without a scheme
|
|
14
|
+
* counts when it reads `user:password@host`. Used to redact those exact strings from text
|
|
15
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
16
|
+
*/
|
|
17
|
+
export declare function credentialsIn(value: string): string[];
|
|
18
|
+
/**
|
|
19
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
20
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
21
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
22
|
+
*/
|
|
23
|
+
export declare function redactCredentials(text: string, credentials: readonly string[]): string;
|
|
24
|
+
/**
|
|
25
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
26
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
27
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
28
|
+
*/
|
|
29
|
+
export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
30
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
31
|
+
export declare function cutForMessage(text: string): string;
|
|
7
32
|
/** Base class for every error originating from this client. */
|
|
8
33
|
export declare class CkanError extends Error {
|
|
9
34
|
constructor(message: string, options?: {
|
|
@@ -20,12 +45,23 @@ export declare class CkanApiError extends CkanError {
|
|
|
20
45
|
readonly url: string;
|
|
21
46
|
readonly method: string;
|
|
22
47
|
readonly body: string;
|
|
48
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
49
|
+
readonly retries: number;
|
|
50
|
+
/**
|
|
51
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
52
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
53
|
+
* else undefined.
|
|
54
|
+
*/
|
|
55
|
+
readonly retryAfterMs: number | undefined;
|
|
23
56
|
constructor(args: {
|
|
24
57
|
status: number;
|
|
25
58
|
url: string;
|
|
26
59
|
method: string;
|
|
27
60
|
body: string;
|
|
28
61
|
detail?: string;
|
|
62
|
+
retries?: number;
|
|
63
|
+
retryAfterMs?: number;
|
|
64
|
+
maxRetryAfterMs?: number;
|
|
29
65
|
});
|
|
30
66
|
/** True for statuses the API documents as transient and retry-able. */
|
|
31
67
|
get isRetryable(): boolean;
|
|
@@ -36,6 +72,13 @@ export declare class CkanNetworkError extends CkanError {
|
|
|
36
72
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
37
73
|
export declare class CkanParseError extends CkanError {
|
|
38
74
|
}
|
|
75
|
+
/**
|
|
76
|
+
* An input the library refuses before sending any request: a client option or a
|
|
77
|
+
* method parameter that breaks one of the rules in `validate.ts`. The message is
|
|
78
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error.
|
|
79
|
+
*/
|
|
80
|
+
export declare class CkanValidationError extends CkanError {
|
|
81
|
+
}
|
|
39
82
|
/**
|
|
40
83
|
* Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
|
|
41
84
|
* `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
|
|
@@ -43,4 +86,3 @@ export declare class CkanParseError extends CkanError {
|
|
|
43
86
|
* (`{"__type": "Validation Error", "rows": ["Invalid integer"]}`).
|
|
44
87
|
*/
|
|
45
88
|
export declare function describeCkanError(error: unknown): string;
|
|
46
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
5
5
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
6
|
-
* A
|
|
6
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
7
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
8
|
+
* value without userinfo is returned unchanged.
|
|
7
9
|
*/
|
|
8
10
|
export function redactUrl(url) {
|
|
9
11
|
let parsed;
|
|
@@ -11,14 +13,70 @@ export function redactUrl(url) {
|
|
|
11
13
|
parsed = new URL(url);
|
|
12
14
|
}
|
|
13
15
|
catch {
|
|
14
|
-
|
|
16
|
+
// A value that doesn't parse can still carry credentials: cut them out by text.
|
|
17
|
+
return redactCredentials(url, credentialsIn(url));
|
|
15
18
|
}
|
|
19
|
+
// `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
|
|
16
20
|
if (parsed.username === "" && parsed.password === "")
|
|
17
|
-
return url;
|
|
21
|
+
return redactCredentials(url, credentialsIn(url));
|
|
18
22
|
parsed.username = "***";
|
|
19
23
|
parsed.password = "";
|
|
20
24
|
return parsed.href;
|
|
21
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
|
|
28
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
|
|
29
|
+
* parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
|
|
30
|
+
* is everything between `://` and the last `@` before the host. A value without a scheme
|
|
31
|
+
* counts when it reads `user:password@host`. Used to redact those exact strings from text
|
|
32
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
33
|
+
*/
|
|
34
|
+
export function credentialsIn(value) {
|
|
35
|
+
const schemeAt = value.indexOf("://");
|
|
36
|
+
const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
|
|
37
|
+
// Without a scheme only the unmistakable `user:password@host` form counts.
|
|
38
|
+
if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
|
|
39
|
+
return [];
|
|
40
|
+
// The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
|
|
41
|
+
const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
|
|
42
|
+
let parses = false;
|
|
43
|
+
try {
|
|
44
|
+
new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
|
|
45
|
+
parses = true;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
49
|
+
}
|
|
50
|
+
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
51
|
+
// first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
|
|
52
|
+
const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
|
|
53
|
+
const end = authority.lastIndexOf("@");
|
|
54
|
+
return end > 0 ? [rest.slice(0, end)] : [];
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
58
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
59
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
60
|
+
*/
|
|
61
|
+
export function redactCredentials(text, credentials) {
|
|
62
|
+
let out = text;
|
|
63
|
+
for (const secret of credentials) {
|
|
64
|
+
if (secret === "")
|
|
65
|
+
continue;
|
|
66
|
+
out = out.split(`${secret}@`).join("***@");
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
72
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
73
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
74
|
+
*/
|
|
75
|
+
export const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
76
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
77
|
+
export function cutForMessage(text) {
|
|
78
|
+
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
|
|
79
|
+
}
|
|
22
80
|
/** Base class for every error originating from this client. */
|
|
23
81
|
export class CkanError extends Error {
|
|
24
82
|
constructor(message, options) {
|
|
@@ -36,16 +94,40 @@ export class CkanApiError extends CkanError {
|
|
|
36
94
|
url;
|
|
37
95
|
method;
|
|
38
96
|
body;
|
|
97
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
98
|
+
retries;
|
|
99
|
+
/**
|
|
100
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
101
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
102
|
+
* else undefined.
|
|
103
|
+
*/
|
|
104
|
+
retryAfterMs;
|
|
39
105
|
constructor(args) {
|
|
40
106
|
// The URL is shown without userinfo: a credential in --base-url must not leak.
|
|
41
107
|
const url = redactUrl(args.url);
|
|
42
|
-
const
|
|
43
|
-
|
|
108
|
+
const parts = [];
|
|
109
|
+
if (args.detail)
|
|
110
|
+
parts.push(cutForMessage(args.detail));
|
|
111
|
+
if (args.retryAfterMs !== undefined) {
|
|
112
|
+
// Say why the retries the caller asked for never ran: the server asked for a
|
|
113
|
+
// wait longer than the engine sleeps, and retrying earlier would land inside it.
|
|
114
|
+
const wait = Math.ceil(args.retryAfterMs / 1000);
|
|
115
|
+
const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
|
|
116
|
+
parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
|
|
117
|
+
}
|
|
118
|
+
const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
|
|
119
|
+
const retries = args.retries ?? 0;
|
|
120
|
+
// Say that the status persisted through retries, so a user knows whether raising
|
|
121
|
+
// --max-retries could help.
|
|
122
|
+
const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
|
|
123
|
+
super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
|
|
44
124
|
this.status = args.status;
|
|
45
125
|
this.url = url;
|
|
46
126
|
this.method = args.method;
|
|
47
127
|
this.body = args.body;
|
|
48
128
|
this.detail = args.detail;
|
|
129
|
+
this.retries = retries;
|
|
130
|
+
this.retryAfterMs = args.retryAfterMs;
|
|
49
131
|
}
|
|
50
132
|
/** True for statuses the API documents as transient and retry-able. */
|
|
51
133
|
get isRetryable() {
|
|
@@ -58,6 +140,13 @@ export class CkanNetworkError extends CkanError {
|
|
|
58
140
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
59
141
|
export class CkanParseError extends CkanError {
|
|
60
142
|
}
|
|
143
|
+
/**
|
|
144
|
+
* An input the library refuses before sending any request: a client option or a
|
|
145
|
+
* method parameter that breaks one of the rules in `validate.ts`. The message is
|
|
146
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error.
|
|
147
|
+
*/
|
|
148
|
+
export class CkanValidationError extends CkanError {
|
|
149
|
+
}
|
|
61
150
|
/**
|
|
62
151
|
* Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
|
|
63
152
|
* `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
|
|
@@ -91,4 +180,3 @@ function solrReason(message) {
|
|
|
91
180
|
return undefined;
|
|
92
181
|
return match[1].replace(/\\+(['"])/g, "$1").trim();
|
|
93
182
|
}
|
|
94
|
-
//# sourceMappingURL=errors.js.map
|
|
@@ -10,13 +10,36 @@ export interface HttpRequest {
|
|
|
10
10
|
timeoutMs?: number;
|
|
11
11
|
/** Hard cap on the response body size in bytes; the request aborts if exceeded. */
|
|
12
12
|
maxResponseBytes?: number;
|
|
13
|
+
/**
|
|
14
|
+
* Aborted when the engine's overall deadline (`timeoutMs`) passes. A transport should
|
|
15
|
+
* stop the request then (`fetch(url, { signal })`); the engine rejects at the deadline
|
|
16
|
+
* either way, and enforces `maxResponseBytes` on the body it gets back, so neither
|
|
17
|
+
* limit depends on the transport.
|
|
18
|
+
*/
|
|
19
|
+
signal?: AbortSignal;
|
|
20
|
+
/**
|
|
21
|
+
* Always `"manual"` from the engine: a transport must not follow redirects. The
|
|
22
|
+
* engine follows them itself and decides per hop whether the `Authorization`
|
|
23
|
+
* header (the base URL's userinfo) goes along (same origin only). A fetch-based
|
|
24
|
+
* transport passes it on: `fetch(url, { redirect })`.
|
|
25
|
+
*/
|
|
26
|
+
redirect?: "manual";
|
|
13
27
|
}
|
|
14
28
|
export interface HttpResponse {
|
|
15
29
|
status: number;
|
|
16
30
|
headers: http.IncomingHttpHeaders;
|
|
17
31
|
body: Buffer;
|
|
32
|
+
/**
|
|
33
|
+
* The URL the response came from, if the transport knows it (fetch's
|
|
34
|
+
* `response.url`). When it is on another origin than the request, the transport
|
|
35
|
+
* followed a redirect itself and the engine rejects the response with a
|
|
36
|
+
* CkanNetworkError.
|
|
37
|
+
*/
|
|
38
|
+
url?: string;
|
|
18
39
|
}
|
|
19
40
|
export type Transport = (request: HttpRequest) => Promise<HttpResponse>;
|
|
41
|
+
/** The message for a body over the size cap, naming the option on both sides. */
|
|
42
|
+
export declare function sizeLimitMessage(maxBytes: number): string;
|
|
20
43
|
/**
|
|
21
44
|
* The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
|
|
22
45
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
@@ -28,4 +51,3 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
|
|
|
28
51
|
* (connection errors, timeouts, malformed URLs).
|
|
29
52
|
*/
|
|
30
53
|
export declare const nodeHttpTransport: Transport;
|
|
31
|
-
//# sourceMappingURL=http.d.ts.map
|
package/dist/src/client/http.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
10
|
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
11
|
+
/** The message for a body over the size cap, naming the option on both sides. */
|
|
12
|
+
export function sizeLimitMessage(maxBytes) {
|
|
13
|
+
return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
|
|
14
|
+
}
|
|
11
15
|
/**
|
|
12
16
|
* The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
|
|
13
17
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
@@ -50,40 +54,49 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
50
54
|
clearTimeout(deadline);
|
|
51
55
|
reject(err);
|
|
52
56
|
};
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
aborted
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
57
|
+
// Node validates the headers synchronously and throws a bare TypeError
|
|
58
|
+
// (ERR_INVALID_CHAR) for one it cannot send; report it as a typed failure.
|
|
59
|
+
let req;
|
|
60
|
+
try {
|
|
61
|
+
req = driver.request(url, {
|
|
62
|
+
method: request.method,
|
|
63
|
+
headers: request.headers,
|
|
64
|
+
}, (res) => {
|
|
65
|
+
const chunks = [];
|
|
66
|
+
let received = 0;
|
|
67
|
+
let aborted = false;
|
|
68
|
+
res.on("data", (chunk) => {
|
|
69
|
+
if (aborted)
|
|
70
|
+
return;
|
|
71
|
+
received += chunk.length;
|
|
72
|
+
if (maxBytes !== undefined && received > maxBytes) {
|
|
73
|
+
aborted = true;
|
|
74
|
+
res.destroy();
|
|
75
|
+
settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
chunks.push(chunk);
|
|
79
|
+
});
|
|
80
|
+
res.on("end", () => {
|
|
81
|
+
if (aborted)
|
|
82
|
+
return;
|
|
83
|
+
settleResolve({
|
|
84
|
+
status: res.statusCode ?? 0,
|
|
85
|
+
headers: res.headers,
|
|
86
|
+
body: Buffer.concat(chunks),
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
res.on("error", (err) => {
|
|
90
|
+
if (aborted)
|
|
91
|
+
return; // we already rejected with the size-cap error
|
|
92
|
+
settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
|
|
79
93
|
});
|
|
80
94
|
});
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
});
|
|
95
|
+
}
|
|
96
|
+
catch (err) {
|
|
97
|
+
reject(new CkanNetworkError(`Invalid request: ${err instanceof Error ? err.message : String(err)}`, { cause: err }));
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
87
100
|
if (request.timeoutMs && request.timeoutMs > 0) {
|
|
88
101
|
// Two complementary guards, both using timeoutMs:
|
|
89
102
|
// - setTimeout: a socket-*inactivity* timeout (no bytes for timeoutMs).
|
|
@@ -100,6 +113,15 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
100
113
|
// Don't let a pending deadline timer keep the event loop alive on its own.
|
|
101
114
|
deadline.unref?.();
|
|
102
115
|
}
|
|
116
|
+
if (request.signal !== undefined) {
|
|
117
|
+
const abort = () => {
|
|
118
|
+
req.destroy(new CkanNetworkError(`Request exceeded the ${request.timeoutMs ?? 0}ms deadline`));
|
|
119
|
+
};
|
|
120
|
+
if (request.signal.aborted)
|
|
121
|
+
abort();
|
|
122
|
+
else
|
|
123
|
+
request.signal.addEventListener("abort", abort, { once: true });
|
|
124
|
+
}
|
|
103
125
|
req.on("error", (err) => {
|
|
104
126
|
// A timeout destroy already passes an CkanNetworkError; don't double-wrap.
|
|
105
127
|
settleReject(err instanceof CkanNetworkError ? err : new CkanNetworkError(err.message, { cause: err }));
|
|
@@ -108,4 +130,3 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
108
130
|
req.write(request.body);
|
|
109
131
|
req.end();
|
|
110
132
|
});
|
|
111
|
-
//# sourceMappingURL=http.js.map
|
|
@@ -1,13 +1,14 @@
|
|
|
1
|
-
export { CkanClient, siteRoot } from "./client.js";
|
|
2
|
-
export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
|
|
1
|
+
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
3
3
|
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
+
export { assertValid } from "./validate.js";
|
|
10
|
+
export type { Problem } from "./validate.js";
|
|
9
11
|
export { PORTALS } from "./portals-list.js";
|
|
10
|
-
export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
|
|
11
|
-
export type { PortalCheck } from "./portals.js";
|
|
12
|
+
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
13
|
+
export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
|
|
12
14
|
export * from "./types.js";
|
|
13
|
-
//# sourceMappingURL=index.d.ts.map
|
package/dist/src/client/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
|
-
export { CkanClient, siteRoot } from "./client.js";
|
|
3
|
-
export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
|
|
2
|
+
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
3
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
+
export { assertValid } from "./validate.js";
|
|
7
8
|
export { PORTALS } from "./portals-list.js";
|
|
8
|
-
export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
|
|
9
|
+
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
9
10
|
export * from "./types.js";
|
|
10
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CkanClient } from "./client.js";
|
|
2
|
+
import { type EngineOptions } from "./engine.js";
|
|
2
3
|
import type { Portal } from "./types.js";
|
|
3
4
|
/**
|
|
4
5
|
* One identity per portal, whatever form its URL was written in: host (without
|
|
@@ -34,4 +35,36 @@ export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
|
|
|
34
35
|
export declare function withCheck(portal: Portal, check: PortalCheck, date: string): Portal;
|
|
35
36
|
/** Run `fn` over `items` with at most `limit` calls in flight; results keep the input order. */
|
|
36
37
|
export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
|
|
37
|
-
|
|
38
|
+
/** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
|
|
39
|
+
export declare const DEFAULT_CHECK_CONCURRENCY = 6;
|
|
40
|
+
/** Options for `checkPortalUrls` and `checkPortals`. */
|
|
41
|
+
export interface CheckPortalsOptions {
|
|
42
|
+
/** Client options for every check; `baseUrl` is replaced by each portal's URL. */
|
|
43
|
+
engineOptions?: EngineOptions;
|
|
44
|
+
/** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
|
|
45
|
+
createClient?: (options: EngineOptions) => CkanClient;
|
|
46
|
+
/** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
|
|
47
|
+
concurrency?: number;
|
|
48
|
+
/**
|
|
49
|
+
* When set, a failed check is repeated once after this pause (ms), so one
|
|
50
|
+
* timeout does not count as down. Unset (the default): a single try.
|
|
51
|
+
*/
|
|
52
|
+
retryDelayMs?: number;
|
|
53
|
+
/** Injectable sleep for the retry pause, for tests. */
|
|
54
|
+
sleep?: (ms: number) => Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Check the CKAN portals at `urls` live (`checkPortal`), with at most
|
|
58
|
+
* `concurrency` checks in flight; results keep the input order. Never throws for
|
|
59
|
+
* one portal: a URL the client refuses becomes a failed check too. Rejects with
|
|
60
|
+
* CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
|
|
61
|
+
*/
|
|
62
|
+
export declare function checkPortalUrls(urls: readonly string[], options?: CheckPortalsOptions): Promise<PortalCheck[]>;
|
|
63
|
+
/**
|
|
64
|
+
* Check every portal in `portals` (the built-in list by default) live and fold
|
|
65
|
+
* each result into the entry (`withCheck`), dated `date` (default: today, UTC).
|
|
66
|
+
* What `ckan portals --check` prints.
|
|
67
|
+
*/
|
|
68
|
+
export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
|
|
69
|
+
date?: string;
|
|
70
|
+
}): Promise<Portal[]>;
|
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
// Known CKAN portals: lookup and a short live check. The list itself is in
|
|
2
2
|
// portals-list.ts, maintained with scripts/update-portals.ts.
|
|
3
|
-
import { siteRoot } from "./client.js";
|
|
4
|
-
import {
|
|
3
|
+
import { CkanClient, siteRoot } from "./client.js";
|
|
4
|
+
import { validateBaseUrl } from "./engine.js";
|
|
5
|
+
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
|
|
6
|
+
import { PORTALS } from "./portals-list.js";
|
|
7
|
+
import { assertValid, intRangeProblem } from "./validate.js";
|
|
5
8
|
/**
|
|
6
9
|
* One identity per portal, whatever form its URL was written in: host (without
|
|
7
10
|
* `www.`) plus path, lower-cased, without scheme, trailing slash or a pasted
|
|
@@ -15,18 +18,29 @@ export function portalKey(url) {
|
|
|
15
18
|
}
|
|
16
19
|
/** A portal from `list` by its id (any case) or by any form of its URL. */
|
|
17
20
|
export function findPortal(idOrUrl, list) {
|
|
21
|
+
if (typeof idOrUrl !== "string")
|
|
22
|
+
throw new CkanValidationError("Invalid portal: Expected a string.");
|
|
23
|
+
if (!Array.isArray(list))
|
|
24
|
+
throw new CkanValidationError("Invalid portal list: Expected an array.");
|
|
18
25
|
const wanted = idOrUrl.trim().toLowerCase();
|
|
19
26
|
const byId = list.find((p) => p.id.toLowerCase() === wanted);
|
|
20
27
|
if (byId)
|
|
21
28
|
return byId;
|
|
22
29
|
let key;
|
|
23
30
|
try {
|
|
24
|
-
key = portalKey(idOrUrl);
|
|
31
|
+
key = portalKey(idOrUrl.trim());
|
|
25
32
|
}
|
|
26
33
|
catch {
|
|
27
34
|
return undefined; // not a URL either
|
|
28
35
|
}
|
|
29
|
-
return list.find((p) =>
|
|
36
|
+
return list.find((p) => {
|
|
37
|
+
try {
|
|
38
|
+
return portalKey(p.url) === key;
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
return false; // an entry without a usable url matches nothing
|
|
42
|
+
}
|
|
43
|
+
});
|
|
30
44
|
}
|
|
31
45
|
/**
|
|
32
46
|
* A short live check of the CKAN portal a client points at. It works when `package_search?rows=0`
|
|
@@ -62,6 +76,17 @@ export async function checkPortal(client) {
|
|
|
62
76
|
}
|
|
63
77
|
return result;
|
|
64
78
|
}
|
|
79
|
+
/**
|
|
80
|
+
* The first error `code` (`ENOTFOUND`, `ECONNREFUSED`, …) along a cause chain: the
|
|
81
|
+
* engine wraps a transport failure in a CkanNetworkError that names the request, so
|
|
82
|
+
* Node's code sits one or more levels down.
|
|
83
|
+
*/
|
|
84
|
+
function errorCode(cause, depth = 0) {
|
|
85
|
+
if (typeof cause !== "object" || cause === null || depth > 5)
|
|
86
|
+
return undefined;
|
|
87
|
+
const code = cause.code;
|
|
88
|
+
return code !== undefined ? code : errorCode(cause.cause, depth + 1);
|
|
89
|
+
}
|
|
65
90
|
/** A short, stable reason for a failed check. */
|
|
66
91
|
function problemOf(err) {
|
|
67
92
|
if (err instanceof CkanApiError) {
|
|
@@ -79,7 +104,7 @@ function problemOf(err) {
|
|
|
79
104
|
return "invalid JSON";
|
|
80
105
|
}
|
|
81
106
|
if (err instanceof CkanNetworkError) {
|
|
82
|
-
const code = err.cause
|
|
107
|
+
const code = errorCode(err.cause);
|
|
83
108
|
if (code === "ENOTFOUND" || code === "EAI_AGAIN")
|
|
84
109
|
return "host not found";
|
|
85
110
|
if (code === "ECONNREFUSED")
|
|
@@ -90,6 +115,10 @@ function problemOf(err) {
|
|
|
90
115
|
return "TLS certificate not verifiable";
|
|
91
116
|
if (/timed out|deadline/.test(message))
|
|
92
117
|
return "timeout";
|
|
118
|
+
// The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
|
|
119
|
+
const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
|
|
120
|
+
if (reason !== undefined)
|
|
121
|
+
return reason.slice(0, 80);
|
|
93
122
|
}
|
|
94
123
|
return message.slice(0, 80);
|
|
95
124
|
}
|
|
@@ -121,4 +150,55 @@ export async function mapLimit(items, limit, fn) {
|
|
|
121
150
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
122
151
|
return results;
|
|
123
152
|
}
|
|
124
|
-
|
|
153
|
+
/** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
|
|
154
|
+
export const DEFAULT_CHECK_CONCURRENCY = 6;
|
|
155
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
156
|
+
/**
|
|
157
|
+
* Check the CKAN portals at `urls` live (`checkPortal`), with at most
|
|
158
|
+
* `concurrency` checks in flight; results keep the input order. Never throws for
|
|
159
|
+
* one portal: a URL the client refuses becomes a failed check too. Rejects with
|
|
160
|
+
* CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
|
|
161
|
+
*/
|
|
162
|
+
export async function checkPortalUrls(urls, options = {}) {
|
|
163
|
+
if (!Array.isArray(urls))
|
|
164
|
+
throw new CkanValidationError("Invalid urls: Expected an array.");
|
|
165
|
+
const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
|
|
166
|
+
const { retryDelayMs } = options;
|
|
167
|
+
if (retryDelayMs !== undefined)
|
|
168
|
+
assertValid("retryDelayMs", retryDelayMs, intRangeProblem(0, Number.MAX_SAFE_INTEGER));
|
|
169
|
+
const createClient = options.createClient ?? ((engine) => new CkanClient(engine));
|
|
170
|
+
const sleep = options.sleep ?? realSleep;
|
|
171
|
+
return mapLimit(urls, concurrency, async (url) => {
|
|
172
|
+
let client;
|
|
173
|
+
try {
|
|
174
|
+
// Checked here, not left to the client: `baseUrl: undefined` selects the
|
|
175
|
+
// Hamburg default, so an entry without a url was checked against Hamburg and
|
|
176
|
+
// reported as working, with Hamburg's dataset count.
|
|
177
|
+
validateBaseUrl(url);
|
|
178
|
+
client = createClient({ ...options.engineOptions, baseUrl: url });
|
|
179
|
+
}
|
|
180
|
+
catch (err) {
|
|
181
|
+
return { ...failedCheck(), problem: problemOf(err) };
|
|
182
|
+
}
|
|
183
|
+
const first = await checkPortal(client);
|
|
184
|
+
if (first.working || retryDelayMs === undefined)
|
|
185
|
+
return first;
|
|
186
|
+
await sleep(retryDelayMs);
|
|
187
|
+
return checkPortal(client);
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* Check every portal in `portals` (the built-in list by default) live and fold
|
|
192
|
+
* each result into the entry (`withCheck`), dated `date` (default: today, UTC).
|
|
193
|
+
* What `ckan portals --check` prints.
|
|
194
|
+
*/
|
|
195
|
+
export async function checkPortals(portals = PORTALS, options = {}) {
|
|
196
|
+
if (!Array.isArray(portals))
|
|
197
|
+
throw new CkanValidationError("Invalid portals: Expected an array.");
|
|
198
|
+
const date = options.date ?? new Date().toISOString().slice(0, 10);
|
|
199
|
+
const checks = await checkPortalUrls(portals.map((p) => p.url), options);
|
|
200
|
+
return portals.map((portal, i) => withCheck(portal, checks[i], date));
|
|
201
|
+
}
|
|
202
|
+
function failedCheck() {
|
|
203
|
+
return { working: false, problem: null, datasets: null, ckanVersion: null, siteTitle: null, siteUrl: null };
|
|
204
|
+
}
|
package/dist/src/client/query.js
CHANGED
package/dist/src/client/types.js
CHANGED