@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +55 -12
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -5
- package/dist/src/cli/io.js +65 -9
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +88 -11
- package/dist/src/cli/run.d.ts +53 -1
- package/dist/src/cli/run.js +239 -8
- package/dist/src/cli/shared.d.ts +17 -1
- package/dist/src/cli/shared.js +33 -3
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +124 -22
- package/dist/src/client/engine.d.ts +62 -9
- package/dist/src/client/engine.js +350 -47
- package/dist/src/client/errors.d.ts +81 -1
- package/dist/src/client/errors.js +171 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +3 -3
- package/dist/src/client/index.js +3 -3
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +39 -4
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +49 -3
- package/package.json +6 -5
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
-
import {
|
|
4
|
+
import { TextDecoder } from "node:util";
|
|
5
|
+
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
5
6
|
import { buildQueryString } from "./query.js";
|
|
6
|
-
import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
|
|
7
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
7
8
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
8
9
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
9
10
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
@@ -19,6 +20,14 @@ export const MAX_REDIRECTS = 10;
|
|
|
19
20
|
export function assertHeaderValue(name, value) {
|
|
20
21
|
return assertValid(name, value, headerValueProblem);
|
|
21
22
|
}
|
|
23
|
+
/** Check an optional function option (`transport`, `sleep`): a function or undefined. */
|
|
24
|
+
function functionOption(name, value, fallback) {
|
|
25
|
+
if (value === undefined)
|
|
26
|
+
return fallback;
|
|
27
|
+
if (typeof value !== "function")
|
|
28
|
+
throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
22
31
|
/** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
|
|
23
32
|
function intOption(name, value, min, max) {
|
|
24
33
|
return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
|
|
@@ -26,8 +35,8 @@ function intOption(name, value, min, max) {
|
|
|
26
35
|
/**
|
|
27
36
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
28
37
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
29
|
-
* once: retrying early would only land inside the window
|
|
30
|
-
* out, and a hostile value must not stall the CLI.
|
|
38
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
39
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
31
40
|
*/
|
|
32
41
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
33
42
|
/** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
|
|
@@ -107,6 +116,39 @@ export function sanitizeServerText(text) {
|
|
|
107
116
|
export function validateBaseUrl(raw) {
|
|
108
117
|
return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
109
118
|
}
|
|
119
|
+
/** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
|
|
120
|
+
function isLoopbackHost(hostname) {
|
|
121
|
+
return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
125
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
126
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
127
|
+
* where nothing leaves the machine.
|
|
128
|
+
*
|
|
129
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
130
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
131
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
132
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
133
|
+
* (once per run, before the first request).
|
|
134
|
+
*/
|
|
135
|
+
export function cleartextProblem(baseUrl, secrets = []) {
|
|
136
|
+
let url;
|
|
137
|
+
try {
|
|
138
|
+
url = new URL(baseUrl);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
|
|
144
|
+
return undefined;
|
|
145
|
+
const userinfo = url.username !== "" || url.password !== "";
|
|
146
|
+
const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
|
|
147
|
+
if (phrases.length === 0)
|
|
148
|
+
return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
|
|
149
|
+
const verb = phrases.length === 1 && !userinfo ? "is" : "are";
|
|
150
|
+
return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
|
|
151
|
+
}
|
|
110
152
|
// The headers the engine sets itself, under the exact keys it uses. They are the
|
|
111
153
|
// only ones that follow a cross-origin redirect.
|
|
112
154
|
const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
@@ -119,9 +161,76 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
|
119
161
|
function engineHeadersOnly(headers) {
|
|
120
162
|
return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
|
|
121
163
|
}
|
|
164
|
+
/** Why `value` is not a usable HttpResponse, or undefined when it is. */
|
|
165
|
+
function responseProblem(value) {
|
|
166
|
+
if (typeof value !== "object" || value === null)
|
|
167
|
+
return "not an object";
|
|
168
|
+
const r = value;
|
|
169
|
+
if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
|
|
170
|
+
return "status is not an HTTP status code";
|
|
171
|
+
}
|
|
172
|
+
if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
|
|
173
|
+
return "headers is not an object";
|
|
174
|
+
if (bodyBytes(r.body) === undefined)
|
|
175
|
+
return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
/**
|
|
179
|
+
* The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
|
|
180
|
+
* Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
|
|
181
|
+
* internal slot, not `instanceof`, so a value from another realm (a vm context, a
|
|
182
|
+
* Jest test) counts. Undefined for anything else.
|
|
183
|
+
*/
|
|
184
|
+
function bodyBytes(value) {
|
|
185
|
+
if (Buffer.isBuffer(value))
|
|
186
|
+
return value;
|
|
187
|
+
if (ArrayBuffer.isView(value))
|
|
188
|
+
return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
189
|
+
const tag = Object.prototype.toString.call(value);
|
|
190
|
+
if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
|
|
191
|
+
return Buffer.from(value);
|
|
192
|
+
return undefined;
|
|
193
|
+
}
|
|
194
|
+
/**
|
|
195
|
+
* The response headers as a plain record with lower-case names. Node's transport
|
|
196
|
+
* lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
|
|
197
|
+
* and a fetch transport naturally returns its `Headers` object, which has no plain
|
|
198
|
+
* properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
|
|
199
|
+
* is copied.
|
|
200
|
+
*/
|
|
201
|
+
function plainHeaders(headers) {
|
|
202
|
+
const h = headers;
|
|
203
|
+
if (typeof h.get === "function" && typeof h.forEach === "function") {
|
|
204
|
+
const record = {};
|
|
205
|
+
h.forEach.call(headers, (value, name) => {
|
|
206
|
+
record[String(name).toLowerCase()] = value;
|
|
207
|
+
});
|
|
208
|
+
return record;
|
|
209
|
+
}
|
|
210
|
+
const record = {};
|
|
211
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
212
|
+
record[name.toLowerCase()] = value;
|
|
213
|
+
}
|
|
214
|
+
return record;
|
|
215
|
+
}
|
|
216
|
+
/** The first value of a header (a repeated one arrives as an array). */
|
|
217
|
+
function headerValue(value) {
|
|
218
|
+
return Array.isArray(value) ? value[0] : value;
|
|
219
|
+
}
|
|
122
220
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
123
221
|
export class RequestEngine {
|
|
124
|
-
|
|
222
|
+
// A real private field (not TypeScript's `private`): util.inspect, console.log and
|
|
223
|
+
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
224
|
+
// logged by accident. Messages use redactUrl.
|
|
225
|
+
#baseUrl;
|
|
226
|
+
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
227
|
+
#credentials;
|
|
228
|
+
/**
|
|
229
|
+
* The forms a server echoes that userinfo back in (the Basic value, the decoded
|
|
230
|
+
* `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
|
|
231
|
+
* password never leaves half of the `user:password` around it.
|
|
232
|
+
*/
|
|
233
|
+
#echoed;
|
|
125
234
|
transport;
|
|
126
235
|
userAgent;
|
|
127
236
|
timeoutMs;
|
|
@@ -136,74 +245,229 @@ export class RequestEngine {
|
|
|
136
245
|
// consumer that injects a custom transport would otherwise get no gating at
|
|
137
246
|
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
138
247
|
// the default.
|
|
139
|
-
this
|
|
140
|
-
this
|
|
248
|
+
this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
249
|
+
this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
|
|
250
|
+
try {
|
|
251
|
+
return [raw, decodeURIComponent(raw)];
|
|
252
|
+
}
|
|
253
|
+
catch {
|
|
254
|
+
return [raw];
|
|
255
|
+
}
|
|
256
|
+
});
|
|
257
|
+
this.#echoed = credentialsIn(this.#baseUrl)
|
|
258
|
+
.flatMap(echoedCredentialForms)
|
|
259
|
+
.sort((a, b) => b.length - a.length);
|
|
260
|
+
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
141
261
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
142
262
|
this.userAgent =
|
|
143
263
|
options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
|
|
144
264
|
this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
|
|
145
265
|
this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
|
|
146
|
-
|
|
266
|
+
// Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
|
|
267
|
+
// after 1 ms, a burst rather than a backoff.
|
|
268
|
+
this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
|
|
147
269
|
this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
|
|
148
270
|
this.maxResponseBytes =
|
|
149
271
|
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
150
|
-
this.sleep = options.sleep
|
|
272
|
+
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
273
|
+
}
|
|
274
|
+
/**
|
|
275
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
276
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
277
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
278
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
279
|
+
* messages built from server text.
|
|
280
|
+
*/
|
|
281
|
+
scrub(text) {
|
|
282
|
+
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
151
283
|
}
|
|
152
|
-
/**
|
|
284
|
+
/**
|
|
285
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
286
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
287
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
288
|
+
* base URL's password.
|
|
289
|
+
*/
|
|
290
|
+
scrubCause(cause, depth = 0) {
|
|
291
|
+
if (this.#credentials.length === 0 || depth > 5)
|
|
292
|
+
return cause;
|
|
293
|
+
if (typeof cause === "string")
|
|
294
|
+
return this.scrub(cause);
|
|
295
|
+
if (!(cause instanceof Error))
|
|
296
|
+
return cause;
|
|
297
|
+
const inner = this.scrubCause(cause.cause, depth + 1);
|
|
298
|
+
const message = this.scrub(cause.message);
|
|
299
|
+
if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
|
|
300
|
+
return cause;
|
|
301
|
+
}
|
|
302
|
+
const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
|
|
303
|
+
copy.name = cause.name;
|
|
304
|
+
const code = cause.code;
|
|
305
|
+
if (code !== undefined)
|
|
306
|
+
Object.assign(copy, { code });
|
|
307
|
+
return copy;
|
|
308
|
+
}
|
|
309
|
+
/**
|
|
310
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
311
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
312
|
+
* Authorization header instead (see basicAuthorization).
|
|
313
|
+
*/
|
|
153
314
|
buildUrl(path, query) {
|
|
315
|
+
return this.composeUrl(path, query, true);
|
|
316
|
+
}
|
|
317
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
318
|
+
composeUrl(path, query, withUserinfo) {
|
|
154
319
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
155
320
|
const qs = query ? buildQueryString(query) : "";
|
|
156
|
-
|
|
321
|
+
const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
|
|
322
|
+
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
323
|
+
}
|
|
324
|
+
/**
|
|
325
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
326
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
327
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
328
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
329
|
+
* becomes a rejection.
|
|
330
|
+
*/
|
|
331
|
+
async callTransport(request) {
|
|
332
|
+
const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
|
|
333
|
+
if (this.timeoutMs === 0)
|
|
334
|
+
return call();
|
|
335
|
+
const controller = new AbortController();
|
|
336
|
+
let timer;
|
|
337
|
+
const deadline = new Promise((_, reject) => {
|
|
338
|
+
timer = setTimeout(() => {
|
|
339
|
+
const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
|
|
340
|
+
controller.abort(err);
|
|
341
|
+
reject(err);
|
|
342
|
+
}, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
|
|
343
|
+
});
|
|
344
|
+
try {
|
|
345
|
+
return await Promise.race([call(controller.signal), deadline]);
|
|
346
|
+
}
|
|
347
|
+
finally {
|
|
348
|
+
clearTimeout(timer);
|
|
349
|
+
}
|
|
157
350
|
}
|
|
158
351
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
159
352
|
async request(method, path, options = { accept: "application/json" }) {
|
|
160
|
-
|
|
353
|
+
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
354
|
+
// Authorization header, per hop, so a redirect to the same origin (relative or
|
|
355
|
+
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
356
|
+
// such as fetch also refuses a URL with credentials outright.
|
|
357
|
+
let url = this.composeUrl(path, options.query, false);
|
|
161
358
|
let headers = {
|
|
162
359
|
Accept: options.accept,
|
|
163
360
|
"User-Agent": this.userAgent,
|
|
164
361
|
};
|
|
362
|
+
const authorization = basicAuthorization(this.#baseUrl);
|
|
363
|
+
if (authorization !== undefined)
|
|
364
|
+
headers["Authorization"] = authorization;
|
|
365
|
+
/** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
|
|
366
|
+
let dropped;
|
|
165
367
|
let attempt = 0;
|
|
166
368
|
let redirects = 0;
|
|
167
369
|
// attempts = initial try + maxRetries (redirects are counted separately)
|
|
168
370
|
for (;;) {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
371
|
+
let response;
|
|
372
|
+
try {
|
|
373
|
+
response = await this.callTransport({
|
|
374
|
+
method,
|
|
375
|
+
url,
|
|
376
|
+
headers,
|
|
377
|
+
timeoutMs: this.timeoutMs,
|
|
378
|
+
redirect: "manual",
|
|
379
|
+
...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
catch (cause) {
|
|
383
|
+
// The default transport rejects with CkanNetworkError only; an injected one may
|
|
384
|
+
// throw anything, and its text may quote the URL with its userinfo. Every
|
|
385
|
+
// failure becomes a CkanNetworkError naming the request (redacted), with the
|
|
386
|
+
// credentials scrubbed from its text and its cause chain; any other CkanError
|
|
387
|
+
// passes through.
|
|
388
|
+
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
389
|
+
throw cause;
|
|
390
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
391
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
392
|
+
cause: this.scrubCause(cause),
|
|
393
|
+
});
|
|
394
|
+
}
|
|
395
|
+
// An injected transport may resolve with anything; a malformed HttpResponse
|
|
396
|
+
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
397
|
+
const invalid = responseProblem(response);
|
|
398
|
+
if (invalid !== undefined) {
|
|
399
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
|
|
400
|
+
}
|
|
401
|
+
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
402
|
+
// did (fetch's default) may have carried the Authorization header to another
|
|
403
|
+
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
404
|
+
const finalUrl = response.url;
|
|
405
|
+
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
406
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
|
|
407
|
+
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
408
|
+
`(HttpRequest.redirect is "manual").`);
|
|
409
|
+
}
|
|
176
410
|
const status = response.status;
|
|
411
|
+
const responseHeaders = plainHeaders(response.headers);
|
|
412
|
+
// fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
|
|
413
|
+
const body = bodyBytes(response.body);
|
|
414
|
+
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
415
|
+
// custom one may have read everything.
|
|
416
|
+
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
417
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
418
|
+
}
|
|
177
419
|
const retryable = status === 429 || status === 503;
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
420
|
+
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
421
|
+
// surfaces at once and names the wait the server asked for.
|
|
422
|
+
const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
|
|
423
|
+
const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
|
|
424
|
+
if (retryable && !tooLong && attempt < this.maxRetries) {
|
|
425
|
+
attempt += 1;
|
|
426
|
+
// Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
|
|
427
|
+
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
428
|
+
// zero-delay burst against a server that had just asked for less load.
|
|
429
|
+
const backoff = this.retryDelayMs * attempt;
|
|
430
|
+
await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
|
|
431
|
+
continue;
|
|
187
432
|
}
|
|
188
433
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
189
|
-
const location =
|
|
434
|
+
const location = headerValue(responseHeaders["location"]);
|
|
190
435
|
const isRedirect = status >= 300 && status < 400;
|
|
191
436
|
const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
|
|
192
437
|
if (nextUrl !== undefined) {
|
|
438
|
+
// Userinfo in a Location is not used: credentials come from the base URL
|
|
439
|
+
// only, as the Authorization header, never from a server.
|
|
440
|
+
nextUrl.username = "";
|
|
441
|
+
nextUrl.password = "";
|
|
442
|
+
// Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
|
|
443
|
+
// never reaches a transport (a custom one may not check the scheme).
|
|
444
|
+
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
445
|
+
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
446
|
+
`for ${method} ${cutForMessage(redactUrl(url))}`);
|
|
447
|
+
}
|
|
193
448
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
194
449
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
195
|
-
// non-credential headers, so
|
|
196
|
-
// re-sent to a different host. The
|
|
197
|
-
//
|
|
198
|
-
//
|
|
199
|
-
|
|
450
|
+
// non-credential headers, so the base URL's Authorization (and any future
|
|
451
|
+
// auth/cookie header) is never re-sent to a different host. The same origin
|
|
452
|
+
// keeps it, whether the Location is relative or absolute. The User-Agent
|
|
453
|
+
// stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
|
|
454
|
+
// sent every request through an http:// base URL without one (and ignored
|
|
455
|
+
// --user-agent).
|
|
456
|
+
const from = new URL(url);
|
|
457
|
+
if (nextUrl.origin !== from.origin) {
|
|
458
|
+
if (headers["Authorization"] !== undefined && dropped === undefined) {
|
|
459
|
+
dropped =
|
|
460
|
+
from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
|
|
461
|
+
? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
|
|
462
|
+
: `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
|
|
463
|
+
}
|
|
200
464
|
headers = engineHeadersOnly(headers);
|
|
201
465
|
}
|
|
202
466
|
url = nextUrl.toString();
|
|
203
467
|
redirects += 1;
|
|
204
468
|
continue;
|
|
205
469
|
}
|
|
206
|
-
const contentType = String(
|
|
470
|
+
const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
|
|
207
471
|
if (status < 200 || status >= 300) {
|
|
208
472
|
// A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
|
|
209
473
|
// redirect this client cannot follow. Past the limit it is a loop; a
|
|
@@ -214,13 +478,16 @@ export class RequestEngine {
|
|
|
214
478
|
const detail = target !== undefined
|
|
215
479
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
216
480
|
: location
|
|
217
|
-
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
481
|
+
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
218
482
|
: "redirect not followed (no Location header)";
|
|
219
|
-
throw new CkanApiError({ status, url, method, body:
|
|
483
|
+
throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
|
|
220
484
|
}
|
|
221
|
-
throw this.toApiError(method, url, status,
|
|
485
|
+
throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
486
|
+
retries: attempt,
|
|
487
|
+
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
488
|
+
});
|
|
222
489
|
}
|
|
223
|
-
return { data:
|
|
490
|
+
return { data: body, contentType, status, url };
|
|
224
491
|
}
|
|
225
492
|
}
|
|
226
493
|
/** Perform a GET expecting JSON and parse it into `T`. */
|
|
@@ -235,15 +502,15 @@ export class RequestEngine {
|
|
|
235
502
|
// path) typically answers with an HTML page and HTTP 200.
|
|
236
503
|
// The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
|
|
237
504
|
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
238
|
-
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
505
|
+
const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
|
|
239
506
|
const message = /json/i.test(mediaType)
|
|
240
|
-
? `Invalid JSON from ${redactUrl(res.url)}`
|
|
241
|
-
: `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
|
|
242
|
-
throw new CkanParseError(message, { cause });
|
|
507
|
+
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
508
|
+
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
509
|
+
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
243
510
|
}
|
|
244
511
|
}
|
|
245
|
-
toApiError(method, url, status, body) {
|
|
246
|
-
const text = body.toString("utf8");
|
|
512
|
+
toApiError(method, url, status, body, hint, retry) {
|
|
513
|
+
const text = this.scrub(body.toString("utf8"));
|
|
247
514
|
let detail;
|
|
248
515
|
try {
|
|
249
516
|
const parsed = JSON.parse(text);
|
|
@@ -274,7 +541,43 @@ export class RequestEngine {
|
|
|
274
541
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
275
542
|
if (detail !== undefined)
|
|
276
543
|
detail = sanitizeServerText(detail);
|
|
277
|
-
|
|
544
|
+
if (hint !== undefined)
|
|
545
|
+
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
546
|
+
return new CkanApiError({
|
|
547
|
+
status,
|
|
548
|
+
url,
|
|
549
|
+
method,
|
|
550
|
+
body: text,
|
|
551
|
+
detail,
|
|
552
|
+
retries: retry.retries,
|
|
553
|
+
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
554
|
+
});
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
558
|
+
function withoutUserinfo(url) {
|
|
559
|
+
const [userinfo] = credentialsIn(url);
|
|
560
|
+
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
564
|
+
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
565
|
+
* userinfo.
|
|
566
|
+
*/
|
|
567
|
+
function basicAuthorization(url) {
|
|
568
|
+
const parsed = new URL(url);
|
|
569
|
+
if (parsed.username === "" && parsed.password === "")
|
|
570
|
+
return undefined;
|
|
571
|
+
const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
|
|
572
|
+
return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
|
|
573
|
+
}
|
|
574
|
+
/** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
|
|
575
|
+
function originOf(url) {
|
|
576
|
+
try {
|
|
577
|
+
return new URL(url).origin;
|
|
578
|
+
}
|
|
579
|
+
catch {
|
|
580
|
+
return url;
|
|
278
581
|
}
|
|
279
582
|
}
|
|
280
583
|
/**
|
|
@@ -290,7 +593,7 @@ function decodeBody(body, contentType, url) {
|
|
|
290
593
|
decoder = new TextDecoder(charset);
|
|
291
594
|
}
|
|
292
595
|
catch {
|
|
293
|
-
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
|
|
596
|
+
throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
|
|
294
597
|
}
|
|
295
598
|
return decoder.decode(body);
|
|
296
599
|
}
|
|
@@ -1,9 +1,61 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
3
3
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
4
|
-
* A
|
|
4
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
5
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
6
|
+
* value without userinfo is returned unchanged.
|
|
5
7
|
*/
|
|
6
8
|
export declare function redactUrl(url: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
13
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
16
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
17
|
+
*/
|
|
18
|
+
export declare function credentialsIn(value: string): string[];
|
|
19
|
+
/**
|
|
20
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
22
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
23
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
24
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
25
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
26
|
+
*/
|
|
27
|
+
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
29
|
+
export declare function redactSecrets(text: string, secrets: readonly string[]): string;
|
|
30
|
+
/**
|
|
31
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
32
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
33
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
34
|
+
*/
|
|
35
|
+
export declare function redactCredentials(text: string, credentials: readonly string[]): string;
|
|
36
|
+
/**
|
|
37
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
38
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
39
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
40
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
41
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
42
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
43
|
+
*/
|
|
44
|
+
export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
45
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
46
|
+
export declare function cutForMessage(text: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
49
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
50
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
51
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
52
|
+
*/
|
|
53
|
+
export declare function cutText(text: string, max: number): string;
|
|
54
|
+
/**
|
|
55
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
56
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
57
|
+
*/
|
|
58
|
+
export declare function toWellFormed(text: string): string;
|
|
7
59
|
/** Base class for every error originating from this client. */
|
|
8
60
|
export declare class CkanError extends Error {
|
|
9
61
|
constructor(message: string, options?: {
|
|
@@ -20,16 +72,44 @@ export declare class CkanApiError extends CkanError {
|
|
|
20
72
|
readonly url: string;
|
|
21
73
|
readonly method: string;
|
|
22
74
|
readonly body: string;
|
|
75
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
76
|
+
readonly retries: number;
|
|
77
|
+
/**
|
|
78
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
79
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
80
|
+
* else undefined.
|
|
81
|
+
*/
|
|
82
|
+
readonly retryAfterMs: number | undefined;
|
|
23
83
|
constructor(args: {
|
|
24
84
|
status: number;
|
|
25
85
|
url: string;
|
|
26
86
|
method: string;
|
|
27
87
|
body: string;
|
|
28
88
|
detail?: string;
|
|
89
|
+
retries?: number;
|
|
90
|
+
retryAfterMs?: number;
|
|
91
|
+
maxRetryAfterMs?: number;
|
|
29
92
|
});
|
|
30
93
|
/** True for statuses the API documents as transient and retry-able. */
|
|
31
94
|
get isRetryable(): boolean;
|
|
32
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
98
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
99
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
100
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
101
|
+
* or undefined when the error carries none.
|
|
102
|
+
*/
|
|
103
|
+
export declare class CkanActionError extends CkanError {
|
|
104
|
+
readonly action: string;
|
|
105
|
+
readonly errorType: string | undefined;
|
|
106
|
+
constructor(message: string, args: {
|
|
107
|
+
action: string;
|
|
108
|
+
errorType?: string | undefined;
|
|
109
|
+
});
|
|
110
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
111
|
+
get isNotFound(): boolean;
|
|
112
|
+
}
|
|
33
113
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
34
114
|
export declare class CkanNetworkError extends CkanError {
|
|
35
115
|
}
|