@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,9 +1,10 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
- import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
4
+ import { TextDecoder } from "node:util";
5
+ import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
5
6
  import { buildQueryString } from "./query.js";
6
- import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
7
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
7
8
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
8
9
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
9
10
  const DEFAULT_USER_AGENT = "ckan-cli";
@@ -19,6 +20,14 @@ export const MAX_REDIRECTS = 10;
19
20
  export function assertHeaderValue(name, value) {
20
21
  return assertValid(name, value, headerValueProblem);
21
22
  }
23
+ /** Check an optional function option (`transport`, `sleep`): a function or undefined. */
24
+ function functionOption(name, value, fallback) {
25
+ if (value === undefined)
26
+ return fallback;
27
+ if (typeof value !== "function")
28
+ throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
29
+ return value;
30
+ }
22
31
  /** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
23
32
  function intOption(name, value, min, max) {
24
33
  return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
@@ -26,8 +35,8 @@ function intOption(name, value, min, max) {
26
35
  /**
27
36
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
28
37
  * server asks for longer, the engine does not retry at all and surfaces the error at
29
- * once: retrying early would only land inside the window the server asked us to wait
30
- * out, and a hostile value must not stall the CLI.
38
+ * once, naming the requested wait: retrying early would only land inside the window
39
+ * the server asked us to wait out, and a hostile value must not stall the CLI.
31
40
  */
32
41
  export const MAX_RETRY_AFTER_MS = 30_000;
33
42
  /** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
@@ -107,6 +116,39 @@ export function sanitizeServerText(text) {
107
116
  export function validateBaseUrl(raw) {
108
117
  return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
109
118
  }
119
+ /** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
120
+ function isLoopbackHost(hostname) {
121
+ return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
122
+ }
123
+ /**
124
+ * Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
125
+ * or `undefined` when they would not: for `https:`, for a
126
+ * URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
127
+ * where nothing leaves the machine.
128
+ *
129
+ * The sentence names the host (`url.host`: host and port, never the userinfo) and what
130
+ * secret travels with the requests: the base URL's credentials when it carries userinfo,
131
+ * and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
132
+ * a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
133
+ * (once per run, before the first request).
134
+ */
135
+ export function cleartextProblem(baseUrl, secrets = []) {
136
+ let url;
137
+ try {
138
+ url = new URL(baseUrl);
139
+ }
140
+ catch {
141
+ return undefined;
142
+ }
143
+ if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
144
+ return undefined;
145
+ const userinfo = url.username !== "" || url.password !== "";
146
+ const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
147
+ if (phrases.length === 0)
148
+ return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
149
+ const verb = phrases.length === 1 && !userinfo ? "is" : "are";
150
+ return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
151
+ }
110
152
  // The headers the engine sets itself, under the exact keys it uses. They are the
111
153
  // only ones that follow a cross-origin redirect.
112
154
  const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
@@ -119,9 +161,76 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
119
161
  function engineHeadersOnly(headers) {
120
162
  return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
121
163
  }
164
+ /** Why `value` is not a usable HttpResponse, or undefined when it is. */
165
+ function responseProblem(value) {
166
+ if (typeof value !== "object" || value === null)
167
+ return "not an object";
168
+ const r = value;
169
+ if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
170
+ return "status is not an HTTP status code";
171
+ }
172
+ if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
173
+ return "headers is not an object";
174
+ if (bodyBytes(r.body) === undefined)
175
+ return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
176
+ return undefined;
177
+ }
178
+ /**
179
+ * The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
180
+ * Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
181
+ * internal slot, not `instanceof`, so a value from another realm (a vm context, a
182
+ * Jest test) counts. Undefined for anything else.
183
+ */
184
+ function bodyBytes(value) {
185
+ if (Buffer.isBuffer(value))
186
+ return value;
187
+ if (ArrayBuffer.isView(value))
188
+ return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
189
+ const tag = Object.prototype.toString.call(value);
190
+ if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
191
+ return Buffer.from(value);
192
+ return undefined;
193
+ }
194
+ /**
195
+ * The response headers as a plain record with lower-case names. Node's transport
196
+ * lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
197
+ * and a fetch transport naturally returns its `Headers` object, which has no plain
198
+ * properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
199
+ * is copied.
200
+ */
201
+ function plainHeaders(headers) {
202
+ const h = headers;
203
+ if (typeof h.get === "function" && typeof h.forEach === "function") {
204
+ const record = {};
205
+ h.forEach.call(headers, (value, name) => {
206
+ record[String(name).toLowerCase()] = value;
207
+ });
208
+ return record;
209
+ }
210
+ const record = {};
211
+ for (const [name, value] of Object.entries(headers)) {
212
+ record[name.toLowerCase()] = value;
213
+ }
214
+ return record;
215
+ }
216
+ /** The first value of a header (a repeated one arrives as an array). */
217
+ function headerValue(value) {
218
+ return Array.isArray(value) ? value[0] : value;
219
+ }
122
220
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
123
221
  export class RequestEngine {
124
- baseUrl;
222
+ // A real private field (not TypeScript's `private`): util.inspect, console.log and
223
+ // JSON.stringify of a client never show it, so a password in the base URL can't be
224
+ // logged by accident. Messages use redactUrl.
225
+ #baseUrl;
226
+ /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
227
+ #credentials;
228
+ /**
229
+ * The forms a server echoes that userinfo back in (the Basic value, the decoded
230
+ * `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
231
+ * password never leaves half of the `user:password` around it.
232
+ */
233
+ #echoed;
125
234
  transport;
126
235
  userAgent;
127
236
  timeoutMs;
@@ -136,74 +245,229 @@ export class RequestEngine {
136
245
  // consumer that injects a custom transport would otherwise get no gating at
137
246
  // all, and could be steered to a non-http(s) scheme. Only undefined selects
138
247
  // the default.
139
- this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
140
- this.transport = options.transport ?? nodeHttpTransport;
248
+ this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
249
+ this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
250
+ try {
251
+ return [raw, decodeURIComponent(raw)];
252
+ }
253
+ catch {
254
+ return [raw];
255
+ }
256
+ });
257
+ this.#echoed = credentialsIn(this.#baseUrl)
258
+ .flatMap(echoedCredentialForms)
259
+ .sort((a, b) => b.length - a.length);
260
+ this.transport = functionOption("transport", options.transport, nodeHttpTransport);
141
261
  // Only undefined selects the default; a blank or unsendable value is refused.
142
262
  this.userAgent =
143
263
  options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
144
264
  this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
145
265
  this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
146
- this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, Number.MAX_SAFE_INTEGER) ?? 200;
266
+ // Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
267
+ // after 1 ms, a burst rather than a backoff.
268
+ this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
147
269
  this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
148
270
  this.maxResponseBytes =
149
271
  intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
150
- this.sleep = options.sleep ?? realSleep;
272
+ this.sleep = functionOption("sleep", options.sleep, realSleep);
273
+ }
274
+ /**
275
+ * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
276
+ * server echoes them back in: `echoedCredentialForms`): server text
277
+ * (an error body or a CKAN error that echoes the request URL) and transport text
278
+ * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
279
+ * messages built from server text.
280
+ */
281
+ scrub(text) {
282
+ return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
151
283
  }
152
- /** Build a fully-qualified URL from a path and optional query parameters. */
284
+ /**
285
+ * A failure as the `cause` of the error the engine raises: the original when its
286
+ * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
287
+ * and the cause chain kept), so logging the error with its causes can't reveal the
288
+ * base URL's password.
289
+ */
290
+ scrubCause(cause, depth = 0) {
291
+ if (this.#credentials.length === 0 || depth > 5)
292
+ return cause;
293
+ if (typeof cause === "string")
294
+ return this.scrub(cause);
295
+ if (!(cause instanceof Error))
296
+ return cause;
297
+ const inner = this.scrubCause(cause.cause, depth + 1);
298
+ const message = this.scrub(cause.message);
299
+ if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
300
+ return cause;
301
+ }
302
+ const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
303
+ copy.name = cause.name;
304
+ const code = cause.code;
305
+ if (code !== undefined)
306
+ Object.assign(copy, { code });
307
+ return copy;
308
+ }
309
+ /**
310
+ * Build a fully-qualified URL from a path and optional query parameters. It keeps
311
+ * the base URL's userinfo; `request()` leaves it out and sends it as an
312
+ * Authorization header instead (see basicAuthorization).
313
+ */
153
314
  buildUrl(path, query) {
315
+ return this.composeUrl(path, query, true);
316
+ }
317
+ /** buildUrl, with or without the base URL's userinfo. */
318
+ composeUrl(path, query, withUserinfo) {
154
319
  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
155
320
  const qs = query ? buildQueryString(query) : "";
156
- return `${this.baseUrl}${normalizedPath}${qs ? `?${qs}` : ""}`;
321
+ const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
322
+ return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
323
+ }
324
+ /**
325
+ * Call the transport under the overall deadline (`timeoutMs`): the request gets an
326
+ * AbortSignal that fires at the deadline, and the call rejects then whether the
327
+ * transport stops or not — a custom transport (fetch, a node:http wrapper) that
328
+ * ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
329
+ * becomes a rejection.
330
+ */
331
+ async callTransport(request) {
332
+ const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
333
+ if (this.timeoutMs === 0)
334
+ return call();
335
+ const controller = new AbortController();
336
+ let timer;
337
+ const deadline = new Promise((_, reject) => {
338
+ timer = setTimeout(() => {
339
+ const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
340
+ controller.abort(err);
341
+ reject(err);
342
+ }, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
343
+ });
344
+ try {
345
+ return await Promise.race([call(controller.signal), deadline]);
346
+ }
347
+ finally {
348
+ clearTimeout(timer);
349
+ }
157
350
  }
158
351
  /** Perform a request with Accept negotiation and transient-error retries. */
159
352
  async request(method, path, options = { accept: "application/json" }) {
160
- let url = this.buildUrl(path, options.query);
353
+ // The transport never sees the base URL's userinfo: the engine sends it as an
354
+ // Authorization header, per hop, so a redirect to the same origin (relative or
355
+ // absolute) keeps it and one to another origin or scheme drops it. A transport
356
+ // such as fetch also refuses a URL with credentials outright.
357
+ let url = this.composeUrl(path, options.query, false);
161
358
  let headers = {
162
359
  Accept: options.accept,
163
360
  "User-Agent": this.userAgent,
164
361
  };
362
+ const authorization = basicAuthorization(this.#baseUrl);
363
+ if (authorization !== undefined)
364
+ headers["Authorization"] = authorization;
365
+ /** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
366
+ let dropped;
165
367
  let attempt = 0;
166
368
  let redirects = 0;
167
369
  // attempts = initial try + maxRetries (redirects are counted separately)
168
370
  for (;;) {
169
- const response = await this.transport({
170
- method,
171
- url,
172
- headers,
173
- timeoutMs: this.timeoutMs,
174
- ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
175
- });
371
+ let response;
372
+ try {
373
+ response = await this.callTransport({
374
+ method,
375
+ url,
376
+ headers,
377
+ timeoutMs: this.timeoutMs,
378
+ redirect: "manual",
379
+ ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
380
+ });
381
+ }
382
+ catch (cause) {
383
+ // The default transport rejects with CkanNetworkError only; an injected one may
384
+ // throw anything, and its text may quote the URL with its userinfo. Every
385
+ // failure becomes a CkanNetworkError naming the request (redacted), with the
386
+ // credentials scrubbed from its text and its cause chain; any other CkanError
387
+ // passes through.
388
+ if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
389
+ throw cause;
390
+ const reason = cause instanceof Error ? cause.message : String(cause);
391
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
392
+ cause: this.scrubCause(cause),
393
+ });
394
+ }
395
+ // An injected transport may resolve with anything; a malformed HttpResponse
396
+ // would otherwise surface below as a raw TypeError, outside the CkanError contract.
397
+ const invalid = responseProblem(response);
398
+ if (invalid !== undefined) {
399
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
400
+ }
401
+ // A transport must not follow redirects itself (`redirect: "manual"`): one that
402
+ // did (fetch's default) may have carried the Authorization header to another
403
+ // host, and the answer is not the one asked for. Reject it when it says so (`url`).
404
+ const finalUrl = response.url;
405
+ if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
406
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
407
+ `(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
408
+ `(HttpRequest.redirect is "manual").`);
409
+ }
176
410
  const status = response.status;
411
+ const responseHeaders = plainHeaders(response.headers);
412
+ // fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
413
+ const body = bodyBytes(response.body);
414
+ // The size cap holds whatever the transport did: the default one aborts early, a
415
+ // custom one may have read everything.
416
+ if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
417
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
418
+ }
177
419
  const retryable = status === 429 || status === 503;
178
- if (retryable && attempt < this.maxRetries) {
179
- // Honour Retry-After; without a usable one, back off linearly. A Retry-After
180
- // beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
181
- const retryAfter = parseRetryAfter(response.headers["retry-after"]);
182
- if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
183
- attempt += 1;
184
- await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
185
- continue;
186
- }
420
+ // A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
421
+ // surfaces at once and names the wait the server asked for.
422
+ const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
423
+ const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
424
+ if (retryable && !tooLong && attempt < this.maxRetries) {
425
+ attempt += 1;
426
+ // Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
427
+ // for less: `Retry-After: 0` or a date in the past turned the retries into a
428
+ // zero-delay burst against a server that had just asked for less load.
429
+ const backoff = this.retryDelayMs * attempt;
430
+ await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
431
+ continue;
187
432
  }
188
433
  // Follow redirects, resolving the Location relative to the current URL.
189
- const location = response.headers["location"];
434
+ const location = headerValue(responseHeaders["location"]);
190
435
  const isRedirect = status >= 300 && status < 400;
191
436
  const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
192
437
  if (nextUrl !== undefined) {
438
+ // Userinfo in a Location is not used: credentials come from the base URL
439
+ // only, as the Authorization header, never from a server.
440
+ nextUrl.username = "";
441
+ nextUrl.password = "";
442
+ // Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
443
+ // never reaches a transport (a custom one may not check the scheme).
444
+ if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
445
+ throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
446
+ `for ${method} ${cutForMessage(redactUrl(url))}`);
447
+ }
193
448
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
194
449
  // port, so an https->http downgrade counts), keep only the engine's own
195
- // non-credential headers, so any future auth/cookie header is never
196
- // re-sent to a different host. The User-Agent stays: Hamburg's own
197
- // http: -> https: hop is cross-origin, and dropping it sent every request
198
- // through an http:// base URL without one (and ignored --user-agent).
199
- if (nextUrl.origin !== new URL(url).origin) {
450
+ // non-credential headers, so the base URL's Authorization (and any future
451
+ // auth/cookie header) is never re-sent to a different host. The same origin
452
+ // keeps it, whether the Location is relative or absolute. The User-Agent
453
+ // stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
454
+ // sent every request through an http:// base URL without one (and ignored
455
+ // --user-agent).
456
+ const from = new URL(url);
457
+ if (nextUrl.origin !== from.origin) {
458
+ if (headers["Authorization"] !== undefined && dropped === undefined) {
459
+ dropped =
460
+ from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
461
+ ? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
462
+ : `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
463
+ }
200
464
  headers = engineHeadersOnly(headers);
201
465
  }
202
466
  url = nextUrl.toString();
203
467
  redirects += 1;
204
468
  continue;
205
469
  }
206
- const contentType = String(response.headers["content-type"] ?? "");
470
+ const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
207
471
  if (status < 200 || status >= 300) {
208
472
  // A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
209
473
  // redirect this client cannot follow. Past the limit it is a loop; a
@@ -214,13 +478,16 @@ export class RequestEngine {
214
478
  const detail = target !== undefined
215
479
  ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
216
480
  : location
217
- ? `redirect to ${sanitizeServerText(location)} not followed`
481
+ ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
218
482
  : "redirect not followed (no Location header)";
219
- throw new CkanApiError({ status, url, method, body: response.body.toString("utf8"), detail });
483
+ throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
220
484
  }
221
- throw this.toApiError(method, url, status, response.body);
485
+ throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
486
+ retries: attempt,
487
+ ...(tooLong ? { retryAfterMs: retryAfter } : {}),
488
+ });
222
489
  }
223
- return { data: response.body, contentType, status, url };
490
+ return { data: body, contentType, status, url };
224
491
  }
225
492
  }
226
493
  /** Perform a GET expecting JSON and parse it into `T`. */
@@ -235,15 +502,15 @@ export class RequestEngine {
235
502
  // path) typically answers with an HTML page and HTTP 200.
236
503
  // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
237
504
  // 8-bit CSI U+009B): sanitise it before it reaches stderr.
238
- const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
505
+ const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
239
506
  const message = /json/i.test(mediaType)
240
- ? `Invalid JSON from ${redactUrl(res.url)}`
241
- : `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
242
- throw new CkanParseError(message, { cause });
507
+ ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
508
+ : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
509
+ throw new CkanParseError(message, { cause: this.scrubCause(cause) });
243
510
  }
244
511
  }
245
- toApiError(method, url, status, body) {
246
- const text = body.toString("utf8");
512
+ toApiError(method, url, status, body, hint, retry) {
513
+ const text = this.scrub(body.toString("utf8"));
247
514
  let detail;
248
515
  try {
249
516
  const parsed = JSON.parse(text);
@@ -274,7 +541,43 @@ export class RequestEngine {
274
541
  // endpoint cannot inject terminal escape sequences via the stderr error message.
275
542
  if (detail !== undefined)
276
543
  detail = sanitizeServerText(detail);
277
- return new CkanApiError({ status, url, method, body: text, detail });
544
+ if (hint !== undefined)
545
+ detail = detail === undefined ? hint : `${detail}; ${hint}`;
546
+ return new CkanApiError({
547
+ status,
548
+ url,
549
+ method,
550
+ body: text,
551
+ detail,
552
+ retries: retry.retries,
553
+ ...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
554
+ });
555
+ }
556
+ }
557
+ /** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
558
+ function withoutUserinfo(url) {
559
+ const [userinfo] = credentialsIn(url);
560
+ return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
561
+ }
562
+ /**
563
+ * The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
564
+ * both percent-decoded, as Node's own http client builds it), or undefined without
565
+ * userinfo.
566
+ */
567
+ function basicAuthorization(url) {
568
+ const parsed = new URL(url);
569
+ if (parsed.username === "" && parsed.password === "")
570
+ return undefined;
571
+ const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
572
+ return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
573
+ }
574
+ /** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
575
+ function originOf(url) {
576
+ try {
577
+ return new URL(url).origin;
578
+ }
579
+ catch {
580
+ return url;
278
581
  }
279
582
  }
280
583
  /**
@@ -290,7 +593,7 @@ function decodeBody(body, contentType, url) {
290
593
  decoder = new TextDecoder(charset);
291
594
  }
292
595
  catch {
293
- throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
596
+ throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
294
597
  }
295
598
  return decoder.decode(body);
296
599
  }
@@ -1,9 +1,61 @@
1
1
  /**
2
2
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
3
3
  * credential in a base URL never reaches an error message, a log or CI output.
4
- * A URL without userinfo, or one that does not parse, is returned unchanged.
4
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
5
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
6
+ * value without userinfo is returned unchanged.
5
7
  */
6
8
  export declare function redactUrl(url: string): string;
9
+ /**
10
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
11
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
12
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
13
+ * search text or a User-Agent as often as a credential, and the base URL always has a
14
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
15
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
16
+ * that echoes the value (usage errors, help), whatever characters the password contains.
17
+ */
18
+ export declare function credentialsIn(value: string): string[];
19
+ /**
20
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
21
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
22
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
23
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
24
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
25
+ * are replaced as exact strings (`redactSecrets`).
26
+ */
27
+ export declare function echoedCredentialForms(userinfo: string): string[];
28
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
29
+ export declare function redactSecrets(text: string, secrets: readonly string[]): string;
30
+ /**
31
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
32
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
33
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
34
+ */
35
+ export declare function redactCredentials(text: string, credentials: readonly string[]): string;
36
+ /**
37
+ * Longest URL or server text (in characters) an error message shows. A long search
38
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
39
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
40
+ * value an own message quotes from a server answer or the user's input (a redirect
41
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
42
+ * this length too, so a library caller's `err.message` stays bounded.
43
+ */
44
+ export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
45
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
46
+ export declare function cutForMessage(text: string): string;
47
+ /**
48
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
49
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
50
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
51
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
52
+ */
53
+ export declare function cutText(text: string, max: number): string;
54
+ /**
55
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
56
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
57
+ */
58
+ export declare function toWellFormed(text: string): string;
7
59
  /** Base class for every error originating from this client. */
8
60
  export declare class CkanError extends Error {
9
61
  constructor(message: string, options?: {
@@ -20,16 +72,44 @@ export declare class CkanApiError extends CkanError {
20
72
  readonly url: string;
21
73
  readonly method: string;
22
74
  readonly body: string;
75
+ /** How many times the engine retried the request before giving up (0 when it did not). */
76
+ readonly retries: number;
77
+ /**
78
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
79
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
80
+ * else undefined.
81
+ */
82
+ readonly retryAfterMs: number | undefined;
23
83
  constructor(args: {
24
84
  status: number;
25
85
  url: string;
26
86
  method: string;
27
87
  body: string;
28
88
  detail?: string;
89
+ retries?: number;
90
+ retryAfterMs?: number;
91
+ maxRetryAfterMs?: number;
29
92
  });
30
93
  /** True for statuses the API documents as transient and retry-able. */
31
94
  get isRetryable(): boolean;
32
95
  }
96
+ /**
97
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
98
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
99
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
100
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
101
+ * or undefined when the error carries none.
102
+ */
103
+ export declare class CkanActionError extends CkanError {
104
+ readonly action: string;
105
+ readonly errorType: string | undefined;
106
+ constructor(message: string, args: {
107
+ action: string;
108
+ errorType?: string | undefined;
109
+ });
110
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
111
+ get isNotFound(): boolean;
112
+ }
33
113
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
34
114
  export declare class CkanNetworkError extends CkanError {
35
115
  }