@maschinenlesbar.org/ckan-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
- import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
4
+ import { TextDecoder } from "node:util";
5
+ import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
5
6
  import { buildQueryString } from "./query.js";
6
- import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
7
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
7
8
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
8
9
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
9
10
  const DEFAULT_USER_AGENT = "ckan-cli";
@@ -19,6 +20,14 @@ export const MAX_REDIRECTS = 10;
19
20
  export function assertHeaderValue(name, value) {
20
21
  return assertValid(name, value, headerValueProblem);
21
22
  }
23
+ /** Check an optional function option (`transport`, `sleep`): a function or undefined. */
24
+ function functionOption(name, value, fallback) {
25
+ if (value === undefined)
26
+ return fallback;
27
+ if (typeof value !== "function")
28
+ throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
29
+ return value;
30
+ }
22
31
  /** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
23
32
  function intOption(name, value, min, max) {
24
33
  return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
@@ -26,8 +35,8 @@ function intOption(name, value, min, max) {
26
35
  /**
27
36
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
28
37
  * server asks for longer, the engine does not retry at all and surfaces the error at
29
- * once: retrying early would only land inside the window the server asked us to wait
30
- * out, and a hostile value must not stall the CLI.
38
+ * once, naming the requested wait: retrying early would only land inside the window
39
+ * the server asked us to wait out, and a hostile value must not stall the CLI.
31
40
  */
32
41
  export const MAX_RETRY_AFTER_MS = 30_000;
33
42
  /** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
@@ -119,9 +128,70 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
119
128
  function engineHeadersOnly(headers) {
120
129
  return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
121
130
  }
131
+ /** Why `value` is not a usable HttpResponse, or undefined when it is. */
132
+ function responseProblem(value) {
133
+ if (typeof value !== "object" || value === null)
134
+ return "not an object";
135
+ const r = value;
136
+ if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
137
+ return "status is not an HTTP status code";
138
+ }
139
+ if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
140
+ return "headers is not an object";
141
+ if (bodyBytes(r.body) === undefined)
142
+ return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
143
+ return undefined;
144
+ }
145
+ /**
146
+ * The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
147
+ * Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
148
+ * internal slot, not `instanceof`, so a value from another realm (a vm context, a
149
+ * Jest test) counts. Undefined for anything else.
150
+ */
151
+ function bodyBytes(value) {
152
+ if (Buffer.isBuffer(value))
153
+ return value;
154
+ if (ArrayBuffer.isView(value))
155
+ return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
156
+ const tag = Object.prototype.toString.call(value);
157
+ if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
158
+ return Buffer.from(value);
159
+ return undefined;
160
+ }
161
+ /**
162
+ * The response headers as a plain record with lower-case names. Node's transport
163
+ * lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
164
+ * and a fetch transport naturally returns its `Headers` object, which has no plain
165
+ * properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
166
+ * is copied.
167
+ */
168
+ function plainHeaders(headers) {
169
+ const h = headers;
170
+ if (typeof h.get === "function" && typeof h.forEach === "function") {
171
+ const record = {};
172
+ h.forEach.call(headers, (value, name) => {
173
+ record[String(name).toLowerCase()] = value;
174
+ });
175
+ return record;
176
+ }
177
+ const record = {};
178
+ for (const [name, value] of Object.entries(headers)) {
179
+ record[name.toLowerCase()] = value;
180
+ }
181
+ return record;
182
+ }
183
+ /** The first value of a header (a repeated one arrives as an array). */
184
+ function headerValue(value) {
185
+ return Array.isArray(value) ? value[0] : value;
186
+ }
122
187
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
123
188
  export class RequestEngine {
124
- baseUrl;
189
+ // A real private field (not TypeScript's `private`): util.inspect, console.log and
190
+ // JSON.stringify of a client never show it, so a password in the base URL can't be
191
+ // logged by accident. Messages use redactUrl.
192
+ #baseUrl;
193
+ /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
194
+ #credentials;
125
195
  transport;
126
196
  userAgent;
127
197
  timeoutMs;
@@ -136,74 +206,225 @@ export class RequestEngine {
136
206
  // consumer that injects a custom transport would otherwise get no gating at
137
207
  // all, and could be steered to a non-http(s) scheme. Only undefined selects
138
208
  // the default.
139
- this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
140
- this.transport = options.transport ?? nodeHttpTransport;
209
+ this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
210
+ this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
211
+ try {
212
+ return [raw, decodeURIComponent(raw)];
213
+ }
214
+ catch {
215
+ return [raw];
216
+ }
217
+ });
218
+ this.transport = functionOption("transport", options.transport, nodeHttpTransport);
141
219
  // Only undefined selects the default; a blank or unsendable value is refused.
142
220
  this.userAgent =
143
221
  options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
144
222
  this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
145
223
  this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
146
- this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, Number.MAX_SAFE_INTEGER) ?? 200;
224
+ // Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
225
+ // after 1 ms, a burst rather than a backoff.
226
+ this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
147
227
  this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
148
228
  this.maxResponseBytes =
149
229
  intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
150
- this.sleep = options.sleep ?? realSleep;
230
+ this.sleep = functionOption("sleep", options.sleep, realSleep);
231
+ }
232
+ /**
233
+ * `text` without the base URL's credentials (raw and percent-decoded): server text
234
+ * (an error body or a CKAN error that echoes the request URL) and transport text
235
+ * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
236
+ * messages built from server text.
237
+ */
238
+ scrub(text) {
239
+ return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
151
240
  }
152
- /** Build a fully-qualified URL from a path and optional query parameters. */
241
+ /**
242
+ * A failure as the `cause` of the error the engine raises: the original when its
243
+ * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
244
+ * and the cause chain kept), so logging the error with its causes can't reveal the
245
+ * base URL's password.
246
+ */
247
+ scrubCause(cause, depth = 0) {
248
+ if (this.#credentials.length === 0 || depth > 5)
249
+ return cause;
250
+ if (typeof cause === "string")
251
+ return this.scrub(cause);
252
+ if (!(cause instanceof Error))
253
+ return cause;
254
+ const inner = this.scrubCause(cause.cause, depth + 1);
255
+ const message = this.scrub(cause.message);
256
+ if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
257
+ return cause;
258
+ }
259
+ const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
260
+ copy.name = cause.name;
261
+ const code = cause.code;
262
+ if (code !== undefined)
263
+ Object.assign(copy, { code });
264
+ return copy;
265
+ }
266
+ /**
267
+ * Build a fully-qualified URL from a path and optional query parameters. It keeps
268
+ * the base URL's userinfo; `request()` leaves it out and sends it as an
269
+ * Authorization header instead (see basicAuthorization).
270
+ */
153
271
  buildUrl(path, query) {
272
+ return this.composeUrl(path, query, true);
273
+ }
274
+ /** buildUrl, with or without the base URL's userinfo. */
275
+ composeUrl(path, query, withUserinfo) {
154
276
  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
155
277
  const qs = query ? buildQueryString(query) : "";
156
- return `${this.baseUrl}${normalizedPath}${qs ? `?${qs}` : ""}`;
278
+ const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
279
+ return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
280
+ }
281
+ /**
282
+ * Call the transport under the overall deadline (`timeoutMs`): the request gets an
283
+ * AbortSignal that fires at the deadline, and the call rejects then whether the
284
+ * transport stops or not — a custom transport (fetch, a node:http wrapper) that
285
+ * ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
286
+ * becomes a rejection.
287
+ */
288
+ async callTransport(request) {
289
+ const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
290
+ if (this.timeoutMs === 0)
291
+ return call();
292
+ const controller = new AbortController();
293
+ let timer;
294
+ const deadline = new Promise((_, reject) => {
295
+ timer = setTimeout(() => {
296
+ const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
297
+ controller.abort(err);
298
+ reject(err);
299
+ }, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
300
+ });
301
+ try {
302
+ return await Promise.race([call(controller.signal), deadline]);
303
+ }
304
+ finally {
305
+ clearTimeout(timer);
306
+ }
157
307
  }
158
308
  /** Perform a request with Accept negotiation and transient-error retries. */
159
309
  async request(method, path, options = { accept: "application/json" }) {
160
- let url = this.buildUrl(path, options.query);
310
+ // The transport never sees the base URL's userinfo: the engine sends it as an
311
+ // Authorization header, per hop, so a redirect to the same origin (relative or
312
+ // absolute) keeps it and one to another origin or scheme drops it. A transport
313
+ // such as fetch also refuses a URL with credentials outright.
314
+ let url = this.composeUrl(path, options.query, false);
161
315
  let headers = {
162
316
  Accept: options.accept,
163
317
  "User-Agent": this.userAgent,
164
318
  };
319
+ const authorization = basicAuthorization(this.#baseUrl);
320
+ if (authorization !== undefined)
321
+ headers["Authorization"] = authorization;
322
+ /** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
323
+ let dropped;
165
324
  let attempt = 0;
166
325
  let redirects = 0;
167
326
  // attempts = initial try + maxRetries (redirects are counted separately)
168
327
  for (;;) {
169
- const response = await this.transport({
170
- method,
171
- url,
172
- headers,
173
- timeoutMs: this.timeoutMs,
174
- ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
175
- });
328
+ let response;
329
+ try {
330
+ response = await this.callTransport({
331
+ method,
332
+ url,
333
+ headers,
334
+ timeoutMs: this.timeoutMs,
335
+ redirect: "manual",
336
+ ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
337
+ });
338
+ }
339
+ catch (cause) {
340
+ // The default transport rejects with CkanNetworkError only; an injected one may
341
+ // throw anything, and its text may quote the URL with its userinfo. Every
342
+ // failure becomes a CkanNetworkError naming the request (redacted), with the
343
+ // credentials scrubbed from its text and its cause chain; any other CkanError
344
+ // passes through.
345
+ if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
346
+ throw cause;
347
+ const reason = cause instanceof Error ? cause.message : String(cause);
348
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
349
+ cause: this.scrubCause(cause),
350
+ });
351
+ }
352
+ // An injected transport may resolve with anything; a malformed HttpResponse
353
+ // would otherwise surface below as a raw TypeError, outside the CkanError contract.
354
+ const invalid = responseProblem(response);
355
+ if (invalid !== undefined) {
356
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
357
+ }
358
+ // A transport must not follow redirects itself (`redirect: "manual"`): one that
359
+ // did (fetch's default) may have carried the Authorization header to another
360
+ // host, and the answer is not the one asked for. Reject it when it says so (`url`).
361
+ const finalUrl = response.url;
362
+ if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
363
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
364
+ `(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
365
+ `(HttpRequest.redirect is "manual").`);
366
+ }
176
367
  const status = response.status;
368
+ const responseHeaders = plainHeaders(response.headers);
369
+ // fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
370
+ const body = bodyBytes(response.body);
371
+ // The size cap holds whatever the transport did: the default one aborts early, a
372
+ // custom one may have read everything.
373
+ if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
374
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
375
+ }
177
376
  const retryable = status === 429 || status === 503;
178
- if (retryable && attempt < this.maxRetries) {
179
- // Honour Retry-After; without a usable one, back off linearly. A Retry-After
180
- // beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
181
- const retryAfter = parseRetryAfter(response.headers["retry-after"]);
182
- if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
183
- attempt += 1;
184
- await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
185
- continue;
186
- }
377
+ // A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
378
+ // surfaces at once and names the wait the server asked for.
379
+ const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
380
+ const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
381
+ if (retryable && !tooLong && attempt < this.maxRetries) {
382
+ attempt += 1;
383
+ // Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
384
+ // for less: `Retry-After: 0` or a date in the past turned the retries into a
385
+ // zero-delay burst against a server that had just asked for less load.
386
+ const backoff = this.retryDelayMs * attempt;
387
+ await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
388
+ continue;
187
389
  }
188
390
  // Follow redirects, resolving the Location relative to the current URL.
189
- const location = response.headers["location"];
391
+ const location = headerValue(responseHeaders["location"]);
190
392
  const isRedirect = status >= 300 && status < 400;
191
393
  const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
192
394
  if (nextUrl !== undefined) {
395
+ // Userinfo in a Location is not used: credentials come from the base URL
396
+ // only, as the Authorization header, never from a server.
397
+ nextUrl.username = "";
398
+ nextUrl.password = "";
399
+ // Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
400
+ // never reaches a transport (a custom one may not check the scheme).
401
+ if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
402
+ throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
403
+ `for ${method} ${cutForMessage(redactUrl(url))}`);
404
+ }
193
405
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
194
406
  // port, so an https->http downgrade counts), keep only the engine's own
195
- // non-credential headers, so any future auth/cookie header is never
196
- // re-sent to a different host. The User-Agent stays: Hamburg's own
197
- // http: -> https: hop is cross-origin, and dropping it sent every request
198
- // through an http:// base URL without one (and ignored --user-agent).
199
- if (nextUrl.origin !== new URL(url).origin) {
407
+ // non-credential headers, so the base URL's Authorization (and any future
408
+ // auth/cookie header) is never re-sent to a different host. The same origin
409
+ // keeps it, whether the Location is relative or absolute. The User-Agent
410
+ // stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
411
+ // sent every request through an http:// base URL without one (and ignored
412
+ // --user-agent).
413
+ const from = new URL(url);
414
+ if (nextUrl.origin !== from.origin) {
415
+ if (headers["Authorization"] !== undefined && dropped === undefined) {
416
+ dropped =
417
+ from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
418
+ ? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
419
+ : `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
420
+ }
200
421
  headers = engineHeadersOnly(headers);
201
422
  }
202
423
  url = nextUrl.toString();
203
424
  redirects += 1;
204
425
  continue;
205
426
  }
206
- const contentType = String(response.headers["content-type"] ?? "");
427
+ const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
207
428
  if (status < 200 || status >= 300) {
208
429
  // A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
209
430
  // redirect this client cannot follow. Past the limit it is a loop; a
@@ -216,11 +437,14 @@ export class RequestEngine {
216
437
  : location
217
438
  ? `redirect to ${sanitizeServerText(location)} not followed`
218
439
  : "redirect not followed (no Location header)";
219
- throw new CkanApiError({ status, url, method, body: response.body.toString("utf8"), detail });
440
+ throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
220
441
  }
221
- throw this.toApiError(method, url, status, response.body);
442
+ throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
443
+ retries: attempt,
444
+ ...(tooLong ? { retryAfterMs: retryAfter } : {}),
445
+ });
222
446
  }
223
- return { data: response.body, contentType, status, url };
447
+ return { data: body, contentType, status, url };
224
448
  }
225
449
  }
226
450
  /** Perform a GET expecting JSON and parse it into `T`. */
@@ -237,13 +461,13 @@ export class RequestEngine {
237
461
  // 8-bit CSI U+009B): sanitise it before it reaches stderr.
238
462
  const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
239
463
  const message = /json/i.test(mediaType)
240
- ? `Invalid JSON from ${redactUrl(res.url)}`
241
- : `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
242
- throw new CkanParseError(message, { cause });
464
+ ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
465
+ : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
466
+ throw new CkanParseError(message, { cause: this.scrubCause(cause) });
243
467
  }
244
468
  }
245
- toApiError(method, url, status, body) {
246
- const text = body.toString("utf8");
469
+ toApiError(method, url, status, body, hint, retry) {
470
+ const text = this.scrub(body.toString("utf8"));
247
471
  let detail;
248
472
  try {
249
473
  const parsed = JSON.parse(text);
@@ -274,7 +498,43 @@ export class RequestEngine {
274
498
  // endpoint cannot inject terminal escape sequences via the stderr error message.
275
499
  if (detail !== undefined)
276
500
  detail = sanitizeServerText(detail);
277
- return new CkanApiError({ status, url, method, body: text, detail });
501
+ if (hint !== undefined)
502
+ detail = detail === undefined ? hint : `${detail}; ${hint}`;
503
+ return new CkanApiError({
504
+ status,
505
+ url,
506
+ method,
507
+ body: text,
508
+ detail,
509
+ retries: retry.retries,
510
+ ...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
511
+ });
512
+ }
513
+ }
514
+ /** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
515
+ function withoutUserinfo(url) {
516
+ const [userinfo] = credentialsIn(url);
517
+ return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
518
+ }
519
+ /**
520
+ * The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
521
+ * both percent-decoded, as Node's own http client builds it), or undefined without
522
+ * userinfo.
523
+ */
524
+ function basicAuthorization(url) {
525
+ const parsed = new URL(url);
526
+ if (parsed.username === "" && parsed.password === "")
527
+ return undefined;
528
+ const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
529
+ return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
530
+ }
531
+ /** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
532
+ function originOf(url) {
533
+ try {
534
+ return new URL(url).origin;
535
+ }
536
+ catch {
537
+ return url;
278
538
  }
279
539
  }
280
540
  /**
@@ -290,7 +550,7 @@ function decodeBody(body, contentType, url) {
290
550
  decoder = new TextDecoder(charset);
291
551
  }
292
552
  catch {
293
- throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
553
+ throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
294
554
  }
295
555
  return decoder.decode(body);
296
556
  }
@@ -1,9 +1,34 @@
1
1
  /**
2
2
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
3
3
  * credential in a base URL never reaches an error message, a log or CI output.
4
- * A URL without userinfo, or one that does not parse, is returned unchanged.
4
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
5
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
6
+ * value without userinfo is returned unchanged.
5
7
  */
6
8
  export declare function redactUrl(url: string): string;
9
+ /**
10
+ * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
11
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
12
+ * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
13
+ * is everything between `://` and the last `@` before the host. A value without a scheme
14
+ * counts when it reads `user:password@host`. Used to redact those exact strings from text
15
+ * that echoes the value (usage errors, help), whatever characters the password contains.
16
+ */
17
+ export declare function credentialsIn(value: string): string[];
18
+ /**
19
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
20
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
21
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
22
+ */
23
+ export declare function redactCredentials(text: string, credentials: readonly string[]): string;
24
+ /**
25
+ * Longest URL or server text (in characters) an error message shows. A long search
26
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
27
+ * log. The error's `url`, `detail` and `body` properties keep the full value.
28
+ */
29
+ export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
30
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
31
+ export declare function cutForMessage(text: string): string;
7
32
  /** Base class for every error originating from this client. */
8
33
  export declare class CkanError extends Error {
9
34
  constructor(message: string, options?: {
@@ -20,12 +45,23 @@ export declare class CkanApiError extends CkanError {
20
45
  readonly url: string;
21
46
  readonly method: string;
22
47
  readonly body: string;
48
+ /** How many times the engine retried the request before giving up (0 when it did not). */
49
+ readonly retries: number;
50
+ /**
51
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
52
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
53
+ * else undefined.
54
+ */
55
+ readonly retryAfterMs: number | undefined;
23
56
  constructor(args: {
24
57
  status: number;
25
58
  url: string;
26
59
  method: string;
27
60
  body: string;
28
61
  detail?: string;
62
+ retries?: number;
63
+ retryAfterMs?: number;
64
+ maxRetryAfterMs?: number;
29
65
  });
30
66
  /** True for statuses the API documents as transient and retry-able. */
31
67
  get isRetryable(): boolean;
@@ -3,7 +3,9 @@
3
3
  /**
4
4
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
5
5
  * credential in a base URL never reaches an error message, a log or CI output.
6
- * A URL without userinfo, or one that does not parse, is returned unchanged.
6
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
7
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
8
+ * value without userinfo is returned unchanged.
7
9
  */
8
10
  export function redactUrl(url) {
9
11
  let parsed;
@@ -11,14 +13,70 @@ export function redactUrl(url) {
11
13
  parsed = new URL(url);
12
14
  }
13
15
  catch {
14
- return url;
16
+ // A value that doesn't parse can still carry credentials: cut them out by text.
17
+ return redactCredentials(url, credentialsIn(url));
15
18
  }
19
+ // `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
16
20
  if (parsed.username === "" && parsed.password === "")
17
- return url;
21
+ return redactCredentials(url, credentialsIn(url));
18
22
  parsed.username = "***";
19
23
  parsed.password = "";
20
24
  return parsed.href;
21
25
  }
26
+ /**
27
+ * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
28
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
29
+ * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
30
+ * is everything between `://` and the last `@` before the host. A value without a scheme
31
+ * counts when it reads `user:password@host`. Used to redact those exact strings from text
32
+ * that echoes the value (usage errors, help), whatever characters the password contains.
33
+ */
34
+ export function credentialsIn(value) {
35
+ const schemeAt = value.indexOf("://");
36
+ const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
37
+ // Without a scheme only the unmistakable `user:password@host` form counts.
38
+ if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
39
+ return [];
40
+ // The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
41
+ const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
42
+ let parses = false;
43
+ try {
44
+ new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
45
+ parses = true;
46
+ }
47
+ catch {
48
+ // Doesn't parse: the password may hold "/", "?", "#" or spaces.
49
+ }
50
+ // In a URL that parses, the userinfo ends at the last "@" of the authority (before the
51
+ // first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
52
+ const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
53
+ const end = authority.lastIndexOf("@");
54
+ return end > 0 ? [rest.slice(0, end)] : [];
55
+ }
56
+ /**
57
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
58
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
59
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
60
+ */
61
+ export function redactCredentials(text, credentials) {
62
+ let out = text;
63
+ for (const secret of credentials) {
64
+ if (secret === "")
65
+ continue;
66
+ out = out.split(`${secret}@`).join("***@");
67
+ }
68
+ return out;
69
+ }
70
+ /**
71
+ * Longest URL or server text (in characters) an error message shows. A long search
72
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
73
+ * log. The error's `url`, `detail` and `body` properties keep the full value.
74
+ */
75
+ export const MAX_MESSAGE_VALUE_LENGTH = 500;
76
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
77
+ export function cutForMessage(text) {
78
+ return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
79
+ }
22
80
  /** Base class for every error originating from this client. */
23
81
  export class CkanError extends Error {
24
82
  constructor(message, options) {
@@ -36,16 +94,40 @@ export class CkanApiError extends CkanError {
36
94
  url;
37
95
  method;
38
96
  body;
97
+ /** How many times the engine retried the request before giving up (0 when it did not). */
98
+ retries;
99
+ /**
100
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
101
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
102
+ * else undefined.
103
+ */
104
+ retryAfterMs;
39
105
  constructor(args) {
40
106
  // The URL is shown without userinfo: a credential in --base-url must not leak.
41
107
  const url = redactUrl(args.url);
42
- const detailPart = args.detail ? `: ${args.detail}` : "";
43
- super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
108
+ const parts = [];
109
+ if (args.detail)
110
+ parts.push(cutForMessage(args.detail));
111
+ if (args.retryAfterMs !== undefined) {
112
+ // Say why the retries the caller asked for never ran: the server asked for a
113
+ // wait longer than the engine sleeps, and retrying earlier would land inside it.
114
+ const wait = Math.ceil(args.retryAfterMs / 1000);
115
+ const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
116
+ parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
117
+ }
118
+ const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
119
+ const retries = args.retries ?? 0;
120
+ // Say that the status persisted through retries, so a user knows whether raising
121
+ // --max-retries could help.
122
+ const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
123
+ super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
44
124
  this.status = args.status;
45
125
  this.url = url;
46
126
  this.method = args.method;
47
127
  this.body = args.body;
48
128
  this.detail = args.detail;
129
+ this.retries = retries;
130
+ this.retryAfterMs = args.retryAfterMs;
49
131
  }
50
132
  /** True for statuses the API documents as transient and retry-able. */
51
133
  get isRetryable() {