@maschinenlesbar.org/ckan-cli 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -14
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +22 -9
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +18 -10
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +2 -1
- package/dist/src/cli/io.d.ts +17 -1
- package/dist/src/cli/io.js +22 -1
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +24 -6
- package/dist/src/cli/run.d.ts +0 -1
- package/dist/src/cli/run.js +9 -2
- package/dist/src/cli/shared.d.ts +28 -14
- package/dist/src/cli/shared.js +78 -40
- package/dist/src/client/client.d.ts +13 -6
- package/dist/src/client/client.js +100 -35
- package/dist/src/client/engine.d.ts +79 -6
- package/dist/src/client/engine.js +188 -74
- package/dist/src/client/errors.d.ts +13 -1
- package/dist/src/client/errors.js +30 -3
- package/dist/src/client/http.d.ts +0 -1
- package/dist/src/client/http.js +44 -36
- package/dist/src/client/index.d.ts +6 -5
- package/dist/src/client/index.js +4 -4
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +50 -3
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +5 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +55 -0
- package/dist/src/client/validate.js +119 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +2 -1
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
|
@@ -1,49 +1,123 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
-
import { nodeHttpTransport } from "./http.js";
|
|
4
|
+
import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
import { buildQueryString } from "./query.js";
|
|
6
|
-
import { CkanApiError,
|
|
6
|
+
import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
|
|
7
|
+
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
7
8
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
8
9
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
9
10
|
const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
11
|
+
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
12
|
+
export const MAX_RETRIES = 10;
|
|
13
|
+
/** Most redirects the engine follows (`maxRedirects`). */
|
|
14
|
+
export const MAX_REDIRECTS = 10;
|
|
10
15
|
/**
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
* `detail` extracted from the body. `JSON.parse` decodes an escaped ESC in an
|
|
14
|
-
* error body into a real ESC byte, so without this a hostile/MITM'd endpoint could
|
|
15
|
-
* drive ANSI/OSC escape sequences into the user's terminal when the message is
|
|
16
|
-
* printed to stderr. The CLI's JSON output is escaped separately
|
|
17
|
-
* (`escapeControlChars` in cli/shared.ts): `JSON.stringify` alone leaves DEL and
|
|
18
|
-
* the C1 range raw. So this only needs to cover text that flows into an error message.
|
|
16
|
+
* Check a value for an HTTP header (headerValueProblem) and return it, or throw
|
|
17
|
+
* CkanValidationError `Invalid <name>: …`.
|
|
19
18
|
*/
|
|
20
|
-
function
|
|
19
|
+
export function assertHeaderValue(name, value) {
|
|
20
|
+
return assertValid(name, value, headerValueProblem);
|
|
21
|
+
}
|
|
22
|
+
/** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
|
|
23
|
+
function intOption(name, value, min, max) {
|
|
24
|
+
return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
28
|
+
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
29
|
+
* once: retrying early would only land inside the window the server asked us to wait
|
|
30
|
+
* out, and a hostile value must not stall the CLI.
|
|
31
|
+
*/
|
|
32
|
+
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
33
|
+
/** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
|
|
34
|
+
const IMF_FIXDATE = /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
|
|
35
|
+
/**
|
|
36
|
+
* Parse a `Retry-After` header into a delay in milliseconds (RFC 9110 §10.2.3):
|
|
37
|
+
* either delay-seconds (`"120"`) or an HTTP-date (`"Wed, 21 Oct 2026 07:28:00 GMT"`,
|
|
38
|
+
* turned into the time left from `now`; a date in the past gives 0).
|
|
39
|
+
*
|
|
40
|
+
* Returns `undefined` when the header is absent or malformed — negative (`"-1"`),
|
|
41
|
+
* fractional (`"1.5"`), padded inside, any other date format — so the caller falls
|
|
42
|
+
* back to its own backoff. The strict patterns matter: `Date.parse` alone would
|
|
43
|
+
* read `"1.5"` as a date in 2001 and retry at once.
|
|
44
|
+
*/
|
|
45
|
+
export function parseRetryAfter(header, now = Date.now()) {
|
|
46
|
+
const value = (Array.isArray(header) ? header[0] : header)?.trim();
|
|
47
|
+
if (value === undefined || value === "")
|
|
48
|
+
return undefined;
|
|
49
|
+
if (/^\d+$/.test(value))
|
|
50
|
+
return Number(value) * 1000;
|
|
51
|
+
if (!IMF_FIXDATE.test(value))
|
|
52
|
+
return undefined;
|
|
53
|
+
const when = Date.parse(value);
|
|
54
|
+
return Number.isNaN(when) ? undefined : Math.max(0, when - now);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* True for the Unicode bidirectional formatting characters: ALM (U+061C), LRM/RLM
|
|
58
|
+
* (U+200E/U+200F), the embeddings and overrides U+202A–U+202E and the isolates
|
|
59
|
+
* U+2066–U+2069. A terminal applies them to the text that follows, so an override
|
|
60
|
+
* in server text can reorder what the user sees ("Trojan Source" spoofing).
|
|
61
|
+
*/
|
|
62
|
+
export function isBidiControl(code) {
|
|
63
|
+
return (code === 0x061c ||
|
|
64
|
+
code === 0x200e ||
|
|
65
|
+
code === 0x200f ||
|
|
66
|
+
(code >= 0x202a && code <= 0x202e) ||
|
|
67
|
+
(code >= 0x2066 && code <= 0x2069));
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Make a string that originates in an attacker-controlled response — the error
|
|
71
|
+
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
72
|
+
* into an error message on stderr:
|
|
73
|
+
*
|
|
74
|
+
* - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
|
|
75
|
+
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
76
|
+
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
77
|
+
* spoofing, title changes, OSC 52 clipboard writes).
|
|
78
|
+
* - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
|
|
79
|
+
* reorder the visible message.
|
|
80
|
+
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
81
|
+
* space and the ends are trimmed, so the text stays on one line and a server
|
|
82
|
+
* cannot forge an `Error:` line of its own.
|
|
83
|
+
*
|
|
84
|
+
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
85
|
+
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
86
|
+
* Written as a char-code filter so no raw control byte appears in this source.
|
|
87
|
+
*/
|
|
88
|
+
export function sanitizeServerText(text) {
|
|
21
89
|
let out = "";
|
|
22
90
|
for (const ch of text) {
|
|
23
91
|
const n = ch.codePointAt(0) ?? 0;
|
|
24
|
-
|
|
92
|
+
const whitespaceControl = n >= 0x09 && n <= 0x0d;
|
|
93
|
+
if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
|
|
25
94
|
continue;
|
|
26
95
|
out += ch;
|
|
27
96
|
}
|
|
28
|
-
return out;
|
|
97
|
+
return out.replace(/\s+/g, " ").trim();
|
|
29
98
|
}
|
|
30
99
|
/**
|
|
31
|
-
*
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
*
|
|
100
|
+
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
101
|
+
* characters, an absolute http(s) URL, no query string or fragment) and return it
|
|
102
|
+
* without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
|
|
103
|
+
* configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
|
|
104
|
+
* constructors call it, so a custom transport never sees a bad base URL; the
|
|
105
|
+
* default transport still re-checks the scheme on every hop.
|
|
35
106
|
*/
|
|
36
|
-
function
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
107
|
+
export function validateBaseUrl(raw) {
|
|
108
|
+
return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
109
|
+
}
|
|
110
|
+
// The headers the engine sets itself, under the exact keys it uses. They are the
|
|
111
|
+
// only ones that follow a cross-origin redirect.
|
|
112
|
+
const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
113
|
+
/**
|
|
114
|
+
* A copy of `headers` with only the engine's own non-credential headers (used on
|
|
115
|
+
* cross-origin redirects). A list of known credential headers is never complete
|
|
116
|
+
* (Proxy-Authorization, X-Auth-Token, ...), so an allowlist is kept instead. A new
|
|
117
|
+
* object, so the one already handed to the transport is not changed.
|
|
118
|
+
*/
|
|
119
|
+
function engineHeadersOnly(headers) {
|
|
120
|
+
return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
|
|
47
121
|
}
|
|
48
122
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
49
123
|
export class RequestEngine {
|
|
@@ -57,18 +131,22 @@ export class RequestEngine {
|
|
|
57
131
|
maxResponseBytes;
|
|
58
132
|
sleep;
|
|
59
133
|
constructor(options = {}) {
|
|
60
|
-
|
|
61
|
-
//
|
|
62
|
-
//
|
|
63
|
-
//
|
|
64
|
-
|
|
134
|
+
// The raw value, checked before the slash strip (the engine glues it into
|
|
135
|
+
// every URL), and here rather than only in the default transport: a library
|
|
136
|
+
// consumer that injects a custom transport would otherwise get no gating at
|
|
137
|
+
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
138
|
+
// the default.
|
|
139
|
+
this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
65
140
|
this.transport = options.transport ?? nodeHttpTransport;
|
|
66
|
-
|
|
67
|
-
this.
|
|
68
|
-
|
|
69
|
-
this.
|
|
70
|
-
this.
|
|
71
|
-
this.
|
|
141
|
+
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
142
|
+
this.userAgent =
|
|
143
|
+
options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
|
|
144
|
+
this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
|
|
145
|
+
this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
|
|
146
|
+
this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, Number.MAX_SAFE_INTEGER) ?? 200;
|
|
147
|
+
this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
|
|
148
|
+
this.maxResponseBytes =
|
|
149
|
+
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
72
150
|
this.sleep = options.sleep ?? realSleep;
|
|
73
151
|
}
|
|
74
152
|
/** Build a fully-qualified URL from a path and optional query parameters. */
|
|
@@ -98,42 +176,49 @@ export class RequestEngine {
|
|
|
98
176
|
const status = response.status;
|
|
99
177
|
const retryable = status === 429 || status === 503;
|
|
100
178
|
if (retryable && attempt < this.maxRetries) {
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
179
|
+
// Honour Retry-After; without a usable one, back off linearly. A Retry-After
|
|
180
|
+
// beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
|
|
181
|
+
const retryAfter = parseRetryAfter(response.headers["retry-after"]);
|
|
182
|
+
if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
|
|
183
|
+
attempt += 1;
|
|
184
|
+
await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
|
|
185
|
+
continue;
|
|
186
|
+
}
|
|
104
187
|
}
|
|
105
188
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
url = next.toString();
|
|
119
|
-
redirects += 1;
|
|
120
|
-
continue;
|
|
189
|
+
const location = response.headers["location"];
|
|
190
|
+
const isRedirect = status >= 300 && status < 400;
|
|
191
|
+
const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
|
|
192
|
+
if (nextUrl !== undefined) {
|
|
193
|
+
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
194
|
+
// port, so an https->http downgrade counts), keep only the engine's own
|
|
195
|
+
// non-credential headers, so any future auth/cookie header is never
|
|
196
|
+
// re-sent to a different host. The User-Agent stays: Hamburg's own
|
|
197
|
+
// http: -> https: hop is cross-origin, and dropping it sent every request
|
|
198
|
+
// through an http:// base URL without one (and ignored --user-agent).
|
|
199
|
+
if (nextUrl.origin !== new URL(url).origin) {
|
|
200
|
+
headers = engineHeadersOnly(headers);
|
|
121
201
|
}
|
|
202
|
+
url = nextUrl.toString();
|
|
203
|
+
redirects += 1;
|
|
204
|
+
continue;
|
|
122
205
|
}
|
|
123
206
|
const contentType = String(response.headers["content-type"] ?? "");
|
|
124
207
|
if (status < 200 || status >= 300) {
|
|
125
|
-
// A 3xx that was not followed
|
|
126
|
-
//
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
208
|
+
// A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
|
|
209
|
+
// redirect this client cannot follow. Past the limit it is a loop; a
|
|
210
|
+
// missing or malformed Location (which `new URL` would have thrown on as
|
|
211
|
+
// an "Unexpected error") is named as it came, sanitised.
|
|
212
|
+
if (isRedirect) {
|
|
213
|
+
const target = resolveLocation(location, url);
|
|
214
|
+
const detail = target !== undefined
|
|
215
|
+
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
216
|
+
: location
|
|
217
|
+
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
218
|
+
: "redirect not followed (no Location header)";
|
|
219
|
+
throw new CkanApiError({ status, url, method, body: response.body.toString("utf8"), detail });
|
|
220
|
+
}
|
|
221
|
+
throw this.toApiError(method, url, status, response.body);
|
|
137
222
|
}
|
|
138
223
|
return { data: response.body, contentType, status, url };
|
|
139
224
|
}
|
|
@@ -141,17 +226,19 @@ export class RequestEngine {
|
|
|
141
226
|
/** Perform a GET expecting JSON and parse it into `T`. */
|
|
142
227
|
async getJson(path, query) {
|
|
143
228
|
const res = await this.request("GET", path, { query, accept: "application/json" });
|
|
144
|
-
const text = res.data.
|
|
229
|
+
const text = decodeBody(res.data, res.contentType, res.url);
|
|
145
230
|
try {
|
|
146
231
|
return JSON.parse(text);
|
|
147
232
|
}
|
|
148
233
|
catch (cause) {
|
|
149
234
|
// Name the URL and what came back: a site that is not a CKAN (or not at this
|
|
150
235
|
// path) typically answers with an HTML page and HTTP 200.
|
|
151
|
-
|
|
236
|
+
// The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
|
|
237
|
+
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
238
|
+
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
152
239
|
const message = /json/i.test(mediaType)
|
|
153
|
-
? `Invalid JSON from ${res.url}`
|
|
154
|
-
: `Expected JSON from ${res.url} but got ${mediaType || "a body that is not JSON"}`;
|
|
240
|
+
? `Invalid JSON from ${redactUrl(res.url)}`
|
|
241
|
+
: `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
|
|
155
242
|
throw new CkanParseError(message, { cause });
|
|
156
243
|
}
|
|
157
244
|
}
|
|
@@ -190,4 +277,31 @@ export class RequestEngine {
|
|
|
190
277
|
return new CkanApiError({ status, url, method, body: text, detail });
|
|
191
278
|
}
|
|
192
279
|
}
|
|
193
|
-
|
|
280
|
+
/**
|
|
281
|
+
* Decode a response body by the charset of its Content-Type (UTF-8 when none is
|
|
282
|
+
* given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
|
|
283
|
+
* that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
|
|
284
|
+
* sends UTF-8; this matters for proxies and mirrors that re-encode.
|
|
285
|
+
*/
|
|
286
|
+
function decodeBody(body, contentType, url) {
|
|
287
|
+
const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
|
|
288
|
+
let decoder;
|
|
289
|
+
try {
|
|
290
|
+
decoder = new TextDecoder(charset);
|
|
291
|
+
}
|
|
292
|
+
catch {
|
|
293
|
+
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
|
|
294
|
+
}
|
|
295
|
+
return decoder.decode(body);
|
|
296
|
+
}
|
|
297
|
+
/** Resolve a Location header against the current URL; undefined if missing or malformed. */
|
|
298
|
+
function resolveLocation(location, base) {
|
|
299
|
+
if (location === undefined || location === "")
|
|
300
|
+
return undefined;
|
|
301
|
+
try {
|
|
302
|
+
return new URL(location, base);
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return undefined;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
3
|
+
* credential in a base URL never reaches an error message, a log or CI output.
|
|
4
|
+
* A URL without userinfo, or one that does not parse, is returned unchanged.
|
|
5
|
+
*/
|
|
6
|
+
export declare function redactUrl(url: string): string;
|
|
1
7
|
/** Base class for every error originating from this client. */
|
|
2
8
|
export declare class CkanError extends Error {
|
|
3
9
|
constructor(message: string, options?: {
|
|
@@ -30,6 +36,13 @@ export declare class CkanNetworkError extends CkanError {
|
|
|
30
36
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
31
37
|
export declare class CkanParseError extends CkanError {
|
|
32
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* An input the library refuses before sending any request: a client option or a
|
|
41
|
+
* method parameter that breaks one of the rules in `validate.ts`. The message is
|
|
42
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error.
|
|
43
|
+
*/
|
|
44
|
+
export declare class CkanValidationError extends CkanError {
|
|
45
|
+
}
|
|
33
46
|
/**
|
|
34
47
|
* Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
|
|
35
48
|
* `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
|
|
@@ -37,4 +50,3 @@ export declare class CkanParseError extends CkanError {
|
|
|
37
50
|
* (`{"__type": "Validation Error", "rows": ["Invalid integer"]}`).
|
|
38
51
|
*/
|
|
39
52
|
export declare function describeCkanError(error: unknown): string;
|
|
40
|
-
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
// Error types raised by the client. Kept free of any I/O so they are trivial to
|
|
2
2
|
// construct in tests and to `instanceof`-check by consumers.
|
|
3
|
+
/**
|
|
4
|
+
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
5
|
+
* credential in a base URL never reaches an error message, a log or CI output.
|
|
6
|
+
* A URL without userinfo, or one that does not parse, is returned unchanged.
|
|
7
|
+
*/
|
|
8
|
+
export function redactUrl(url) {
|
|
9
|
+
let parsed;
|
|
10
|
+
try {
|
|
11
|
+
parsed = new URL(url);
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
return url;
|
|
15
|
+
}
|
|
16
|
+
if (parsed.username === "" && parsed.password === "")
|
|
17
|
+
return url;
|
|
18
|
+
parsed.username = "***";
|
|
19
|
+
parsed.password = "";
|
|
20
|
+
return parsed.href;
|
|
21
|
+
}
|
|
3
22
|
/** Base class for every error originating from this client. */
|
|
4
23
|
export class CkanError extends Error {
|
|
5
24
|
constructor(message, options) {
|
|
@@ -18,10 +37,12 @@ export class CkanApiError extends CkanError {
|
|
|
18
37
|
method;
|
|
19
38
|
body;
|
|
20
39
|
constructor(args) {
|
|
40
|
+
// The URL is shown without userinfo: a credential in --base-url must not leak.
|
|
41
|
+
const url = redactUrl(args.url);
|
|
21
42
|
const detailPart = args.detail ? `: ${args.detail}` : "";
|
|
22
|
-
super(`HTTP ${args.status} for ${args.method} ${
|
|
43
|
+
super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
|
|
23
44
|
this.status = args.status;
|
|
24
|
-
this.url =
|
|
45
|
+
this.url = url;
|
|
25
46
|
this.method = args.method;
|
|
26
47
|
this.body = args.body;
|
|
27
48
|
this.detail = args.detail;
|
|
@@ -37,6 +58,13 @@ export class CkanNetworkError extends CkanError {
|
|
|
37
58
|
/** The response body could not be parsed as the expected JSON shape. */
|
|
38
59
|
export class CkanParseError extends CkanError {
|
|
39
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* An input the library refuses before sending any request: a client option or a
|
|
63
|
+
* method parameter that breaks one of the rules in `validate.ts`. The message is
|
|
64
|
+
* `Invalid <name>: <reason>`. The CLI reports it as a usage error.
|
|
65
|
+
*/
|
|
66
|
+
export class CkanValidationError extends CkanError {
|
|
67
|
+
}
|
|
40
68
|
/**
|
|
41
69
|
* Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
|
|
42
70
|
* `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
|
|
@@ -70,4 +98,3 @@ function solrReason(message) {
|
|
|
70
98
|
return undefined;
|
|
71
99
|
return match[1].replace(/\\+(['"])/g, "$1").trim();
|
|
72
100
|
}
|
|
73
|
-
//# sourceMappingURL=errors.js.map
|
package/dist/src/client/http.js
CHANGED
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
// `http.createServer` in the test-suite.
|
|
8
8
|
import http from "node:http";
|
|
9
9
|
import https from "node:https";
|
|
10
|
-
import { CkanNetworkError } from "./errors.js";
|
|
10
|
+
import { CkanNetworkError, redactUrl } from "./errors.js";
|
|
11
11
|
/**
|
|
12
12
|
* The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
|
|
13
13
|
* prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
|
|
@@ -24,14 +24,14 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
24
24
|
url = new URL(request.url);
|
|
25
25
|
}
|
|
26
26
|
catch {
|
|
27
|
-
reject(new CkanNetworkError(`Invalid URL: ${request.url}`));
|
|
27
|
+
reject(new CkanNetworkError(`Invalid URL: ${redactUrl(request.url)}`));
|
|
28
28
|
return;
|
|
29
29
|
}
|
|
30
30
|
// Only http/https are supported. Reject anything else up front with a clear,
|
|
31
31
|
// typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
|
|
32
32
|
// (and so this never reaches the file:/ftp:/etc. drivers).
|
|
33
33
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
34
|
-
reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${request.url}`));
|
|
34
|
+
reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${redactUrl(request.url)}`));
|
|
35
35
|
return;
|
|
36
36
|
}
|
|
37
37
|
const isHttps = url.protocol === "https:";
|
|
@@ -50,40 +50,49 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
50
50
|
clearTimeout(deadline);
|
|
51
51
|
reject(err);
|
|
52
52
|
};
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
aborted
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
53
|
+
// Node validates the headers synchronously and throws a bare TypeError
|
|
54
|
+
// (ERR_INVALID_CHAR) for one it cannot send; report it as a typed failure.
|
|
55
|
+
let req;
|
|
56
|
+
try {
|
|
57
|
+
req = driver.request(url, {
|
|
58
|
+
method: request.method,
|
|
59
|
+
headers: request.headers,
|
|
60
|
+
}, (res) => {
|
|
61
|
+
const chunks = [];
|
|
62
|
+
let received = 0;
|
|
63
|
+
let aborted = false;
|
|
64
|
+
res.on("data", (chunk) => {
|
|
65
|
+
if (aborted)
|
|
66
|
+
return;
|
|
67
|
+
received += chunk.length;
|
|
68
|
+
if (maxBytes !== undefined && received > maxBytes) {
|
|
69
|
+
aborted = true;
|
|
70
|
+
res.destroy();
|
|
71
|
+
settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
chunks.push(chunk);
|
|
75
|
+
});
|
|
76
|
+
res.on("end", () => {
|
|
77
|
+
if (aborted)
|
|
78
|
+
return;
|
|
79
|
+
settleResolve({
|
|
80
|
+
status: res.statusCode ?? 0,
|
|
81
|
+
headers: res.headers,
|
|
82
|
+
body: Buffer.concat(chunks),
|
|
83
|
+
});
|
|
84
|
+
});
|
|
85
|
+
res.on("error", (err) => {
|
|
86
|
+
if (aborted)
|
|
87
|
+
return; // we already rejected with the size-cap error
|
|
88
|
+
settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
|
|
79
89
|
});
|
|
80
90
|
});
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
});
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
reject(new CkanNetworkError(`Invalid request: ${err instanceof Error ? err.message : String(err)}`, { cause: err }));
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
87
96
|
if (request.timeoutMs && request.timeoutMs > 0) {
|
|
88
97
|
// Two complementary guards, both using timeoutMs:
|
|
89
98
|
// - setTimeout: a socket-*inactivity* timeout (no bytes for timeoutMs).
|
|
@@ -108,4 +117,3 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
|
|
|
108
117
|
req.write(request.body);
|
|
109
118
|
req.end();
|
|
110
119
|
});
|
|
111
|
-
//# sourceMappingURL=http.js.map
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
export { CkanClient, siteRoot } from "./client.js";
|
|
2
|
-
export { RequestEngine, DEFAULT_BASE_URL } from "./engine.js";
|
|
2
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
3
3
|
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, } from "./errors.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
|
|
9
|
+
export { assertValid } from "./validate.js";
|
|
10
|
+
export type { Problem } from "./validate.js";
|
|
9
11
|
export { PORTALS } from "./portals-list.js";
|
|
10
|
-
export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
|
|
11
|
-
export type { PortalCheck } from "./portals.js";
|
|
12
|
+
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
13
|
+
export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
|
|
12
14
|
export * from "./types.js";
|
|
13
|
-
//# sourceMappingURL=index.d.ts.map
|
package/dist/src/client/index.js
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
2
|
export { CkanClient, siteRoot } from "./client.js";
|
|
3
|
-
export { RequestEngine, DEFAULT_BASE_URL } from "./engine.js";
|
|
3
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, } from "./errors.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
|
|
7
|
+
export { assertValid } from "./validate.js";
|
|
7
8
|
export { PORTALS } from "./portals-list.js";
|
|
8
|
-
export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
|
|
9
|
+
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
9
10
|
export * from "./types.js";
|
|
10
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { CkanClient } from "./client.js";
|
|
2
|
+
import type { EngineOptions } from "./engine.js";
|
|
2
3
|
import type { Portal } from "./types.js";
|
|
3
4
|
/**
|
|
4
5
|
* One identity per portal, whatever form its URL was written in: host (without
|
|
@@ -34,4 +35,36 @@ export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
|
|
|
34
35
|
export declare function withCheck(portal: Portal, check: PortalCheck, date: string): Portal;
|
|
35
36
|
/** Run `fn` over `items` with at most `limit` calls in flight; results keep the input order. */
|
|
36
37
|
export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
|
|
37
|
-
|
|
38
|
+
/** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
|
|
39
|
+
export declare const DEFAULT_CHECK_CONCURRENCY = 6;
|
|
40
|
+
/** Options for `checkPortalUrls` and `checkPortals`. */
|
|
41
|
+
export interface CheckPortalsOptions {
|
|
42
|
+
/** Client options for every check; `baseUrl` is replaced by each portal's URL. */
|
|
43
|
+
engineOptions?: EngineOptions;
|
|
44
|
+
/** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
|
|
45
|
+
createClient?: (options: EngineOptions) => CkanClient;
|
|
46
|
+
/** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
|
|
47
|
+
concurrency?: number;
|
|
48
|
+
/**
|
|
49
|
+
* When set, a failed check is repeated once after this pause (ms), so one
|
|
50
|
+
* timeout does not count as down. Unset (the default): a single try.
|
|
51
|
+
*/
|
|
52
|
+
retryDelayMs?: number;
|
|
53
|
+
/** Injectable sleep for the retry pause, for tests. */
|
|
54
|
+
sleep?: (ms: number) => Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* Check the CKAN portals at `urls` live (`checkPortal`), with at most
|
|
58
|
+
* `concurrency` checks in flight; results keep the input order. Never throws for
|
|
59
|
+
* one portal: a URL the client refuses becomes a failed check too. Rejects with
|
|
60
|
+
* CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
|
|
61
|
+
*/
|
|
62
|
+
export declare function checkPortalUrls(urls: readonly string[], options?: CheckPortalsOptions): Promise<PortalCheck[]>;
|
|
63
|
+
/**
|
|
64
|
+
* Check every portal in `portals` (the built-in list by default) live and fold
|
|
65
|
+
* each result into the entry (`withCheck`), dated `date` (default: today, UTC).
|
|
66
|
+
* What `ckan portals --check` prints.
|
|
67
|
+
*/
|
|
68
|
+
export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
|
|
69
|
+
date?: string;
|
|
70
|
+
}): Promise<Portal[]>;
|