@maschinenlesbar.org/ckan-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/io.d.ts +2 -1
- package/dist/src/cli/io.js +6 -2
- package/dist/src/cli/program.js +16 -9
- package/dist/src/cli/run.d.ts +27 -0
- package/dist/src/cli/run.js +82 -1
- package/dist/src/cli/shared.d.ts +8 -0
- package/dist/src/cli/shared.js +16 -0
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +123 -22
- package/dist/src/client/engine.d.ts +48 -9
- package/dist/src/client/engine.js +305 -45
- package/dist/src/client/errors.d.ts +37 -1
- package/dist/src/client/errors.js +87 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +2 -2
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +38 -3
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +47 -1
- package/package.json +3 -3
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
-
import {
|
|
4
|
+
import { TextDecoder } from "node:util";
|
|
5
|
+
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
5
6
|
import { buildQueryString } from "./query.js";
|
|
6
|
-
import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
|
|
7
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
|
|
7
8
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
8
9
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
9
10
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
@@ -19,6 +20,14 @@ export const MAX_REDIRECTS = 10;
|
|
|
19
20
|
export function assertHeaderValue(name, value) {
|
|
20
21
|
return assertValid(name, value, headerValueProblem);
|
|
21
22
|
}
|
|
23
|
+
/** Check an optional function option (`transport`, `sleep`): a function or undefined. */
|
|
24
|
+
function functionOption(name, value, fallback) {
|
|
25
|
+
if (value === undefined)
|
|
26
|
+
return fallback;
|
|
27
|
+
if (typeof value !== "function")
|
|
28
|
+
throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
22
31
|
/** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
|
|
23
32
|
function intOption(name, value, min, max) {
|
|
24
33
|
return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
|
|
@@ -26,8 +35,8 @@ function intOption(name, value, min, max) {
|
|
|
26
35
|
/**
|
|
27
36
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
28
37
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
29
|
-
* once: retrying early would only land inside the window
|
|
30
|
-
* out, and a hostile value must not stall the CLI.
|
|
38
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
39
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
31
40
|
*/
|
|
32
41
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
33
42
|
/** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
|
|
@@ -119,9 +128,70 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
|
119
128
|
function engineHeadersOnly(headers) {
|
|
120
129
|
return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
|
|
121
130
|
}
|
|
131
|
+
/** Why `value` is not a usable HttpResponse, or undefined when it is. */
|
|
132
|
+
function responseProblem(value) {
|
|
133
|
+
if (typeof value !== "object" || value === null)
|
|
134
|
+
return "not an object";
|
|
135
|
+
const r = value;
|
|
136
|
+
if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
|
|
137
|
+
return "status is not an HTTP status code";
|
|
138
|
+
}
|
|
139
|
+
if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
|
|
140
|
+
return "headers is not an object";
|
|
141
|
+
if (bodyBytes(r.body) === undefined)
|
|
142
|
+
return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
|
|
147
|
+
* Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
|
|
148
|
+
* internal slot, not `instanceof`, so a value from another realm (a vm context, a
|
|
149
|
+
* Jest test) counts. Undefined for anything else.
|
|
150
|
+
*/
|
|
151
|
+
function bodyBytes(value) {
|
|
152
|
+
if (Buffer.isBuffer(value))
|
|
153
|
+
return value;
|
|
154
|
+
if (ArrayBuffer.isView(value))
|
|
155
|
+
return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
156
|
+
const tag = Object.prototype.toString.call(value);
|
|
157
|
+
if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
|
|
158
|
+
return Buffer.from(value);
|
|
159
|
+
return undefined;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The response headers as a plain record with lower-case names. Node's transport
|
|
163
|
+
* lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
|
|
164
|
+
* and a fetch transport naturally returns its `Headers` object, which has no plain
|
|
165
|
+
* properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
|
|
166
|
+
* is copied.
|
|
167
|
+
*/
|
|
168
|
+
function plainHeaders(headers) {
|
|
169
|
+
const h = headers;
|
|
170
|
+
if (typeof h.get === "function" && typeof h.forEach === "function") {
|
|
171
|
+
const record = {};
|
|
172
|
+
h.forEach.call(headers, (value, name) => {
|
|
173
|
+
record[String(name).toLowerCase()] = value;
|
|
174
|
+
});
|
|
175
|
+
return record;
|
|
176
|
+
}
|
|
177
|
+
const record = {};
|
|
178
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
179
|
+
record[name.toLowerCase()] = value;
|
|
180
|
+
}
|
|
181
|
+
return record;
|
|
182
|
+
}
|
|
183
|
+
/** The first value of a header (a repeated one arrives as an array). */
|
|
184
|
+
function headerValue(value) {
|
|
185
|
+
return Array.isArray(value) ? value[0] : value;
|
|
186
|
+
}
|
|
122
187
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
123
188
|
export class RequestEngine {
|
|
124
|
-
|
|
189
|
+
// A real private field (not TypeScript's `private`): util.inspect, console.log and
|
|
190
|
+
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
191
|
+
// logged by accident. Messages use redactUrl.
|
|
192
|
+
#baseUrl;
|
|
193
|
+
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
194
|
+
#credentials;
|
|
125
195
|
transport;
|
|
126
196
|
userAgent;
|
|
127
197
|
timeoutMs;
|
|
@@ -136,74 +206,225 @@ export class RequestEngine {
|
|
|
136
206
|
// consumer that injects a custom transport would otherwise get no gating at
|
|
137
207
|
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
138
208
|
// the default.
|
|
139
|
-
this
|
|
140
|
-
this
|
|
209
|
+
this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
210
|
+
this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
|
|
211
|
+
try {
|
|
212
|
+
return [raw, decodeURIComponent(raw)];
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return [raw];
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
141
219
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
142
220
|
this.userAgent =
|
|
143
221
|
options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
|
|
144
222
|
this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
|
|
145
223
|
this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
|
|
146
|
-
|
|
224
|
+
// Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
|
|
225
|
+
// after 1 ms, a burst rather than a backoff.
|
|
226
|
+
this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
|
|
147
227
|
this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
|
|
148
228
|
this.maxResponseBytes =
|
|
149
229
|
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
150
|
-
this.sleep = options.sleep
|
|
230
|
+
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* `text` without the base URL's credentials (raw and percent-decoded): server text
|
|
234
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
235
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
236
|
+
* messages built from server text.
|
|
237
|
+
*/
|
|
238
|
+
scrub(text) {
|
|
239
|
+
return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
|
|
151
240
|
}
|
|
152
|
-
/**
|
|
241
|
+
/**
|
|
242
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
243
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
244
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
245
|
+
* base URL's password.
|
|
246
|
+
*/
|
|
247
|
+
scrubCause(cause, depth = 0) {
|
|
248
|
+
if (this.#credentials.length === 0 || depth > 5)
|
|
249
|
+
return cause;
|
|
250
|
+
if (typeof cause === "string")
|
|
251
|
+
return this.scrub(cause);
|
|
252
|
+
if (!(cause instanceof Error))
|
|
253
|
+
return cause;
|
|
254
|
+
const inner = this.scrubCause(cause.cause, depth + 1);
|
|
255
|
+
const message = this.scrub(cause.message);
|
|
256
|
+
if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
|
|
257
|
+
return cause;
|
|
258
|
+
}
|
|
259
|
+
const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
|
|
260
|
+
copy.name = cause.name;
|
|
261
|
+
const code = cause.code;
|
|
262
|
+
if (code !== undefined)
|
|
263
|
+
Object.assign(copy, { code });
|
|
264
|
+
return copy;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
268
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
269
|
+
* Authorization header instead (see basicAuthorization).
|
|
270
|
+
*/
|
|
153
271
|
buildUrl(path, query) {
|
|
272
|
+
return this.composeUrl(path, query, true);
|
|
273
|
+
}
|
|
274
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
275
|
+
composeUrl(path, query, withUserinfo) {
|
|
154
276
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
155
277
|
const qs = query ? buildQueryString(query) : "";
|
|
156
|
-
|
|
278
|
+
const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
|
|
279
|
+
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
283
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
284
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
285
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
286
|
+
* becomes a rejection.
|
|
287
|
+
*/
|
|
288
|
+
async callTransport(request) {
|
|
289
|
+
const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
|
|
290
|
+
if (this.timeoutMs === 0)
|
|
291
|
+
return call();
|
|
292
|
+
const controller = new AbortController();
|
|
293
|
+
let timer;
|
|
294
|
+
const deadline = new Promise((_, reject) => {
|
|
295
|
+
timer = setTimeout(() => {
|
|
296
|
+
const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
|
|
297
|
+
controller.abort(err);
|
|
298
|
+
reject(err);
|
|
299
|
+
}, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
|
|
300
|
+
});
|
|
301
|
+
try {
|
|
302
|
+
return await Promise.race([call(controller.signal), deadline]);
|
|
303
|
+
}
|
|
304
|
+
finally {
|
|
305
|
+
clearTimeout(timer);
|
|
306
|
+
}
|
|
157
307
|
}
|
|
158
308
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
159
309
|
async request(method, path, options = { accept: "application/json" }) {
|
|
160
|
-
|
|
310
|
+
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
311
|
+
// Authorization header, per hop, so a redirect to the same origin (relative or
|
|
312
|
+
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
313
|
+
// such as fetch also refuses a URL with credentials outright.
|
|
314
|
+
let url = this.composeUrl(path, options.query, false);
|
|
161
315
|
let headers = {
|
|
162
316
|
Accept: options.accept,
|
|
163
317
|
"User-Agent": this.userAgent,
|
|
164
318
|
};
|
|
319
|
+
const authorization = basicAuthorization(this.#baseUrl);
|
|
320
|
+
if (authorization !== undefined)
|
|
321
|
+
headers["Authorization"] = authorization;
|
|
322
|
+
/** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
|
|
323
|
+
let dropped;
|
|
165
324
|
let attempt = 0;
|
|
166
325
|
let redirects = 0;
|
|
167
326
|
// attempts = initial try + maxRetries (redirects are counted separately)
|
|
168
327
|
for (;;) {
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
328
|
+
let response;
|
|
329
|
+
try {
|
|
330
|
+
response = await this.callTransport({
|
|
331
|
+
method,
|
|
332
|
+
url,
|
|
333
|
+
headers,
|
|
334
|
+
timeoutMs: this.timeoutMs,
|
|
335
|
+
redirect: "manual",
|
|
336
|
+
...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
catch (cause) {
|
|
340
|
+
// The default transport rejects with CkanNetworkError only; an injected one may
|
|
341
|
+
// throw anything, and its text may quote the URL with its userinfo. Every
|
|
342
|
+
// failure becomes a CkanNetworkError naming the request (redacted), with the
|
|
343
|
+
// credentials scrubbed from its text and its cause chain; any other CkanError
|
|
344
|
+
// passes through.
|
|
345
|
+
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
346
|
+
throw cause;
|
|
347
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
348
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
349
|
+
cause: this.scrubCause(cause),
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
// An injected transport may resolve with anything; a malformed HttpResponse
|
|
353
|
+
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
354
|
+
const invalid = responseProblem(response);
|
|
355
|
+
if (invalid !== undefined) {
|
|
356
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
|
|
357
|
+
}
|
|
358
|
+
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
359
|
+
// did (fetch's default) may have carried the Authorization header to another
|
|
360
|
+
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
361
|
+
const finalUrl = response.url;
|
|
362
|
+
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
363
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
|
|
364
|
+
`(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
|
|
365
|
+
`(HttpRequest.redirect is "manual").`);
|
|
366
|
+
}
|
|
176
367
|
const status = response.status;
|
|
368
|
+
const responseHeaders = plainHeaders(response.headers);
|
|
369
|
+
// fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
|
|
370
|
+
const body = bodyBytes(response.body);
|
|
371
|
+
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
372
|
+
// custom one may have read everything.
|
|
373
|
+
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
374
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
375
|
+
}
|
|
177
376
|
const retryable = status === 429 || status === 503;
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
377
|
+
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
378
|
+
// surfaces at once and names the wait the server asked for.
|
|
379
|
+
const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
|
|
380
|
+
const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
|
|
381
|
+
if (retryable && !tooLong && attempt < this.maxRetries) {
|
|
382
|
+
attempt += 1;
|
|
383
|
+
// Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
|
|
384
|
+
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
385
|
+
// zero-delay burst against a server that had just asked for less load.
|
|
386
|
+
const backoff = this.retryDelayMs * attempt;
|
|
387
|
+
await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
|
|
388
|
+
continue;
|
|
187
389
|
}
|
|
188
390
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
189
|
-
const location =
|
|
391
|
+
const location = headerValue(responseHeaders["location"]);
|
|
190
392
|
const isRedirect = status >= 300 && status < 400;
|
|
191
393
|
const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
|
|
192
394
|
if (nextUrl !== undefined) {
|
|
395
|
+
// Userinfo in a Location is not used: credentials come from the base URL
|
|
396
|
+
// only, as the Authorization header, never from a server.
|
|
397
|
+
nextUrl.username = "";
|
|
398
|
+
nextUrl.password = "";
|
|
399
|
+
// Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
|
|
400
|
+
// never reaches a transport (a custom one may not check the scheme).
|
|
401
|
+
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
402
|
+
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
|
|
403
|
+
`for ${method} ${cutForMessage(redactUrl(url))}`);
|
|
404
|
+
}
|
|
193
405
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
194
406
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
195
|
-
// non-credential headers, so
|
|
196
|
-
// re-sent to a different host. The
|
|
197
|
-
//
|
|
198
|
-
//
|
|
199
|
-
|
|
407
|
+
// non-credential headers, so the base URL's Authorization (and any future
|
|
408
|
+
// auth/cookie header) is never re-sent to a different host. The same origin
|
|
409
|
+
// keeps it, whether the Location is relative or absolute. The User-Agent
|
|
410
|
+
// stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
|
|
411
|
+
// sent every request through an http:// base URL without one (and ignored
|
|
412
|
+
// --user-agent).
|
|
413
|
+
const from = new URL(url);
|
|
414
|
+
if (nextUrl.origin !== from.origin) {
|
|
415
|
+
if (headers["Authorization"] !== undefined && dropped === undefined) {
|
|
416
|
+
dropped =
|
|
417
|
+
from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
|
|
418
|
+
? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
|
|
419
|
+
: `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
|
|
420
|
+
}
|
|
200
421
|
headers = engineHeadersOnly(headers);
|
|
201
422
|
}
|
|
202
423
|
url = nextUrl.toString();
|
|
203
424
|
redirects += 1;
|
|
204
425
|
continue;
|
|
205
426
|
}
|
|
206
|
-
const contentType = String(
|
|
427
|
+
const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
|
|
207
428
|
if (status < 200 || status >= 300) {
|
|
208
429
|
// A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
|
|
209
430
|
// redirect this client cannot follow. Past the limit it is a loop; a
|
|
@@ -216,11 +437,14 @@ export class RequestEngine {
|
|
|
216
437
|
: location
|
|
217
438
|
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
218
439
|
: "redirect not followed (no Location header)";
|
|
219
|
-
throw new CkanApiError({ status, url, method, body:
|
|
440
|
+
throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
|
|
220
441
|
}
|
|
221
|
-
throw this.toApiError(method, url, status,
|
|
442
|
+
throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
443
|
+
retries: attempt,
|
|
444
|
+
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
445
|
+
});
|
|
222
446
|
}
|
|
223
|
-
return { data:
|
|
447
|
+
return { data: body, contentType, status, url };
|
|
224
448
|
}
|
|
225
449
|
}
|
|
226
450
|
/** Perform a GET expecting JSON and parse it into `T`. */
|
|
@@ -237,13 +461,13 @@ export class RequestEngine {
|
|
|
237
461
|
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
238
462
|
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
239
463
|
const message = /json/i.test(mediaType)
|
|
240
|
-
? `Invalid JSON from ${redactUrl(res.url)}`
|
|
241
|
-
: `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
|
|
242
|
-
throw new CkanParseError(message, { cause });
|
|
464
|
+
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
465
|
+
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
466
|
+
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
243
467
|
}
|
|
244
468
|
}
|
|
245
|
-
toApiError(method, url, status, body) {
|
|
246
|
-
const text = body.toString("utf8");
|
|
469
|
+
toApiError(method, url, status, body, hint, retry) {
|
|
470
|
+
const text = this.scrub(body.toString("utf8"));
|
|
247
471
|
let detail;
|
|
248
472
|
try {
|
|
249
473
|
const parsed = JSON.parse(text);
|
|
@@ -274,7 +498,43 @@ export class RequestEngine {
|
|
|
274
498
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
275
499
|
if (detail !== undefined)
|
|
276
500
|
detail = sanitizeServerText(detail);
|
|
277
|
-
|
|
501
|
+
if (hint !== undefined)
|
|
502
|
+
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
503
|
+
return new CkanApiError({
|
|
504
|
+
status,
|
|
505
|
+
url,
|
|
506
|
+
method,
|
|
507
|
+
body: text,
|
|
508
|
+
detail,
|
|
509
|
+
retries: retry.retries,
|
|
510
|
+
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
515
|
+
function withoutUserinfo(url) {
|
|
516
|
+
const [userinfo] = credentialsIn(url);
|
|
517
|
+
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
521
|
+
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
522
|
+
* userinfo.
|
|
523
|
+
*/
|
|
524
|
+
function basicAuthorization(url) {
|
|
525
|
+
const parsed = new URL(url);
|
|
526
|
+
if (parsed.username === "" && parsed.password === "")
|
|
527
|
+
return undefined;
|
|
528
|
+
const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
|
|
529
|
+
return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
|
|
530
|
+
}
|
|
531
|
+
/** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
|
|
532
|
+
function originOf(url) {
|
|
533
|
+
try {
|
|
534
|
+
return new URL(url).origin;
|
|
535
|
+
}
|
|
536
|
+
catch {
|
|
537
|
+
return url;
|
|
278
538
|
}
|
|
279
539
|
}
|
|
280
540
|
/**
|
|
@@ -290,7 +550,7 @@ function decodeBody(body, contentType, url) {
|
|
|
290
550
|
decoder = new TextDecoder(charset);
|
|
291
551
|
}
|
|
292
552
|
catch {
|
|
293
|
-
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
|
|
553
|
+
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
|
|
294
554
|
}
|
|
295
555
|
return decoder.decode(body);
|
|
296
556
|
}
|
|
@@ -1,9 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
3
3
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
4
|
-
* A
|
|
4
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
5
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
6
|
+
* value without userinfo is returned unchanged.
|
|
5
7
|
*/
|
|
6
8
|
export declare function redactUrl(url: string): string;
|
|
9
|
+
/**
|
|
10
|
+
* The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
|
|
12
|
+
* parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
|
|
13
|
+
* is everything between `://` and the last `@` before the host. A value without a scheme
|
|
14
|
+
* counts when it reads `user:password@host`. Used to redact those exact strings from text
|
|
15
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
16
|
+
*/
|
|
17
|
+
export declare function credentialsIn(value: string): string[];
|
|
18
|
+
/**
|
|
19
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
20
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
21
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
22
|
+
*/
|
|
23
|
+
export declare function redactCredentials(text: string, credentials: readonly string[]): string;
|
|
24
|
+
/**
|
|
25
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
26
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
27
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
28
|
+
*/
|
|
29
|
+
export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
30
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
31
|
+
export declare function cutForMessage(text: string): string;
|
|
7
32
|
/** Base class for every error originating from this client. */
|
|
8
33
|
export declare class CkanError extends Error {
|
|
9
34
|
constructor(message: string, options?: {
|
|
@@ -20,12 +45,23 @@ export declare class CkanApiError extends CkanError {
|
|
|
20
45
|
readonly url: string;
|
|
21
46
|
readonly method: string;
|
|
22
47
|
readonly body: string;
|
|
48
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
49
|
+
readonly retries: number;
|
|
50
|
+
/**
|
|
51
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
52
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
53
|
+
* else undefined.
|
|
54
|
+
*/
|
|
55
|
+
readonly retryAfterMs: number | undefined;
|
|
23
56
|
constructor(args: {
|
|
24
57
|
status: number;
|
|
25
58
|
url: string;
|
|
26
59
|
method: string;
|
|
27
60
|
body: string;
|
|
28
61
|
detail?: string;
|
|
62
|
+
retries?: number;
|
|
63
|
+
retryAfterMs?: number;
|
|
64
|
+
maxRetryAfterMs?: number;
|
|
29
65
|
});
|
|
30
66
|
/** True for statuses the API documents as transient and retry-able. */
|
|
31
67
|
get isRetryable(): boolean;
|
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
/**
|
|
4
4
|
* Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
|
|
5
5
|
* credential in a base URL never reaches an error message, a log or CI output.
|
|
6
|
-
* A
|
|
6
|
+
* A value that does not parse as a URL (a port typo, an unencoded `#` in the
|
|
7
|
+
* password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
|
|
8
|
+
* value without userinfo is returned unchanged.
|
|
7
9
|
*/
|
|
8
10
|
export function redactUrl(url) {
|
|
9
11
|
let parsed;
|
|
@@ -11,14 +13,70 @@ export function redactUrl(url) {
|
|
|
11
13
|
parsed = new URL(url);
|
|
12
14
|
}
|
|
13
15
|
catch {
|
|
14
|
-
|
|
16
|
+
// A value that doesn't parse can still carry credentials: cut them out by text.
|
|
17
|
+
return redactCredentials(url, credentialsIn(url));
|
|
15
18
|
}
|
|
19
|
+
// `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
|
|
16
20
|
if (parsed.username === "" && parsed.password === "")
|
|
17
|
-
return url;
|
|
21
|
+
return redactCredentials(url, credentialsIn(url));
|
|
18
22
|
parsed.username = "***";
|
|
19
23
|
parsed.password = "";
|
|
20
24
|
return parsed.href;
|
|
21
25
|
}
|
|
26
|
+
/**
|
|
27
|
+
* The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
|
|
28
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
|
|
29
|
+
* parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
|
|
30
|
+
* is everything between `://` and the last `@` before the host. A value without a scheme
|
|
31
|
+
* counts when it reads `user:password@host`. Used to redact those exact strings from text
|
|
32
|
+
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
33
|
+
*/
|
|
34
|
+
export function credentialsIn(value) {
|
|
35
|
+
const schemeAt = value.indexOf("://");
|
|
36
|
+
const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
|
|
37
|
+
// Without a scheme only the unmistakable `user:password@host` form counts.
|
|
38
|
+
if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
|
|
39
|
+
return [];
|
|
40
|
+
// The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
|
|
41
|
+
const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
|
|
42
|
+
let parses = false;
|
|
43
|
+
try {
|
|
44
|
+
new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
|
|
45
|
+
parses = true;
|
|
46
|
+
}
|
|
47
|
+
catch {
|
|
48
|
+
// Doesn't parse: the password may hold "/", "?", "#" or spaces.
|
|
49
|
+
}
|
|
50
|
+
// In a URL that parses, the userinfo ends at the last "@" of the authority (before the
|
|
51
|
+
// first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
|
|
52
|
+
const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
|
|
53
|
+
const end = authority.lastIndexOf("@");
|
|
54
|
+
return end > 0 ? [rest.slice(0, end)] : [];
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
58
|
+
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
59
|
+
* passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
|
|
60
|
+
*/
|
|
61
|
+
export function redactCredentials(text, credentials) {
|
|
62
|
+
let out = text;
|
|
63
|
+
for (const secret of credentials) {
|
|
64
|
+
if (secret === "")
|
|
65
|
+
continue;
|
|
66
|
+
out = out.split(`${secret}@`).join("***@");
|
|
67
|
+
}
|
|
68
|
+
return out;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Longest URL or server text (in characters) an error message shows. A long search
|
|
72
|
+
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
73
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
74
|
+
*/
|
|
75
|
+
export const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
76
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
77
|
+
export function cutForMessage(text) {
|
|
78
|
+
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
|
|
79
|
+
}
|
|
22
80
|
/** Base class for every error originating from this client. */
|
|
23
81
|
export class CkanError extends Error {
|
|
24
82
|
constructor(message, options) {
|
|
@@ -36,16 +94,40 @@ export class CkanApiError extends CkanError {
|
|
|
36
94
|
url;
|
|
37
95
|
method;
|
|
38
96
|
body;
|
|
97
|
+
/** How many times the engine retried the request before giving up (0 when it did not). */
|
|
98
|
+
retries;
|
|
99
|
+
/**
|
|
100
|
+
* The wait the server asked for in `Retry-After` (milliseconds) when it was longer
|
|
101
|
+
* than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
|
|
102
|
+
* else undefined.
|
|
103
|
+
*/
|
|
104
|
+
retryAfterMs;
|
|
39
105
|
constructor(args) {
|
|
40
106
|
// The URL is shown without userinfo: a credential in --base-url must not leak.
|
|
41
107
|
const url = redactUrl(args.url);
|
|
42
|
-
const
|
|
43
|
-
|
|
108
|
+
const parts = [];
|
|
109
|
+
if (args.detail)
|
|
110
|
+
parts.push(cutForMessage(args.detail));
|
|
111
|
+
if (args.retryAfterMs !== undefined) {
|
|
112
|
+
// Say why the retries the caller asked for never ran: the server asked for a
|
|
113
|
+
// wait longer than the engine sleeps, and retrying earlier would land inside it.
|
|
114
|
+
const wait = Math.ceil(args.retryAfterMs / 1000);
|
|
115
|
+
const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
|
|
116
|
+
parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
|
|
117
|
+
}
|
|
118
|
+
const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
|
|
119
|
+
const retries = args.retries ?? 0;
|
|
120
|
+
// Say that the status persisted through retries, so a user knows whether raising
|
|
121
|
+
// --max-retries could help.
|
|
122
|
+
const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
|
|
123
|
+
super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
|
|
44
124
|
this.status = args.status;
|
|
45
125
|
this.url = url;
|
|
46
126
|
this.method = args.method;
|
|
47
127
|
this.body = args.body;
|
|
48
128
|
this.detail = args.detail;
|
|
129
|
+
this.retries = retries;
|
|
130
|
+
this.retryAfterMs = args.retryAfterMs;
|
|
49
131
|
}
|
|
50
132
|
/** True for statuses the API documents as transient and retry-able. */
|
|
51
133
|
get isRetryable() {
|