@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +21 -5
  2. package/dist/src/cli/commands/catalogue.d.ts +1 -2
  3. package/dist/src/cli/commands/catalogue.js +23 -15
  4. package/dist/src/cli/commands/portal.d.ts +0 -1
  5. package/dist/src/cli/commands/portal.js +13 -12
  6. package/dist/src/cli/index.d.ts +0 -1
  7. package/dist/src/cli/index.js +0 -1
  8. package/dist/src/cli/io.d.ts +2 -2
  9. package/dist/src/cli/io.js +6 -3
  10. package/dist/src/cli/program.d.ts +0 -1
  11. package/dist/src/cli/program.js +17 -11
  12. package/dist/src/cli/run.d.ts +27 -1
  13. package/dist/src/cli/run.js +90 -2
  14. package/dist/src/cli/shared.d.ts +18 -15
  15. package/dist/src/cli/shared.js +45 -54
  16. package/dist/src/client/client.d.ts +30 -9
  17. package/dist/src/client/client.js +154 -37
  18. package/dist/src/client/engine.d.ts +74 -13
  19. package/dist/src/client/engine.js +341 -71
  20. package/dist/src/client/errors.d.ts +44 -2
  21. package/dist/src/client/errors.js +94 -6
  22. package/dist/src/client/http.d.ts +23 -1
  23. package/dist/src/client/http.js +54 -33
  24. package/dist/src/client/index.d.ts +7 -6
  25. package/dist/src/client/index.js +5 -5
  26. package/dist/src/client/portals-list.d.ts +0 -1
  27. package/dist/src/client/portals-list.js +0 -1
  28. package/dist/src/client/portals.d.ts +35 -2
  29. package/dist/src/client/portals.js +86 -6
  30. package/dist/src/client/query.d.ts +0 -1
  31. package/dist/src/client/query.js +0 -1
  32. package/dist/src/client/types.d.ts +0 -1
  33. package/dist/src/client/types.js +0 -1
  34. package/dist/src/client/validate.d.ts +67 -0
  35. package/dist/src/client/validate.js +165 -0
  36. package/dist/src/index.d.ts +0 -1
  37. package/dist/src/index.js +0 -1
  38. package/package.json +4 -3
  39. package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
  40. package/dist/src/cli/commands/catalogue.js.map +0 -1
  41. package/dist/src/cli/commands/portal.d.ts.map +0 -1
  42. package/dist/src/cli/commands/portal.js.map +0 -1
  43. package/dist/src/cli/index.d.ts.map +0 -1
  44. package/dist/src/cli/index.js.map +0 -1
  45. package/dist/src/cli/io.d.ts.map +0 -1
  46. package/dist/src/cli/io.js.map +0 -1
  47. package/dist/src/cli/program.d.ts.map +0 -1
  48. package/dist/src/cli/program.js.map +0 -1
  49. package/dist/src/cli/run.d.ts.map +0 -1
  50. package/dist/src/cli/run.js.map +0 -1
  51. package/dist/src/cli/shared.d.ts.map +0 -1
  52. package/dist/src/cli/shared.js.map +0 -1
  53. package/dist/src/client/client.d.ts.map +0 -1
  54. package/dist/src/client/client.js.map +0 -1
  55. package/dist/src/client/engine.d.ts.map +0 -1
  56. package/dist/src/client/engine.js.map +0 -1
  57. package/dist/src/client/errors.d.ts.map +0 -1
  58. package/dist/src/client/errors.js.map +0 -1
  59. package/dist/src/client/http.d.ts.map +0 -1
  60. package/dist/src/client/http.js.map +0 -1
  61. package/dist/src/client/index.d.ts.map +0 -1
  62. package/dist/src/client/index.js.map +0 -1
  63. package/dist/src/client/portals-list.d.ts.map +0 -1
  64. package/dist/src/client/portals-list.js.map +0 -1
  65. package/dist/src/client/portals.d.ts.map +0 -1
  66. package/dist/src/client/portals.js.map +0 -1
  67. package/dist/src/client/query.d.ts.map +0 -1
  68. package/dist/src/client/query.js.map +0 -1
  69. package/dist/src/client/types.d.ts.map +0 -1
  70. package/dist/src/client/types.js.map +0 -1
  71. package/dist/src/index.d.ts.map +0 -1
  72. package/dist/src/index.js.map +0 -1
@@ -13,35 +13,60 @@ export interface EngineOptions {
13
13
  baseUrl?: string;
14
14
  /** Swappable transport. Defaults to the built-in node http/https transport. */
15
15
  transport?: Transport;
16
- /** Value of the User-Agent header. */
16
+ /**
17
+ * Value of the User-Agent header. Must be sendable (see `assertHeaderValue`):
18
+ * not blank, no control characters but tab, nothing above U+00FF.
19
+ */
17
20
  userAgent?: string;
18
- /** Per-request timeout in milliseconds (0 disables; capped at `MAX_TIMEOUT_MS`, 2^31 - 1 ms). */
21
+ /**
22
+ * Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
23
+ * It covers the whole response, body included, and the engine enforces it for every
24
+ * transport (the request also gets an AbortSignal that fires at the deadline).
25
+ */
19
26
  timeoutMs?: number;
20
27
  /**
21
- * Number of automatic retries for transient (429/503) responses. Each waits the
22
- * response's `Retry-After` (up to `MAX_RETRY_AFTER_MS`; a longer one is not
23
- * retried), or else `retryDelayMs * attempt`.
28
+ * Number of automatic retries for transient (429/503) responses; a reset or other
29
+ * network error is not retried. Each waits `retryDelayMs * attempt`, or the
30
+ * response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
31
+ * one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
24
32
  */
25
33
  maxRetries?: number;
26
- /** Base backoff between retries in milliseconds (grows linearly); used without a Retry-After. */
34
+ /**
35
+ * Base backoff between retries in milliseconds (grows linearly), 0 to
36
+ * `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
37
+ * `Retry-After`: the header can lengthen a wait, never shorten it.
38
+ */
27
39
  retryDelayMs?: number;
28
- /** Number of HTTP redirects (301/302/303/307/308) to follow. Defaults to 5. */
40
+ /** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
29
41
  maxRedirects?: number;
30
42
  /**
31
43
  * Hard cap on response body size in bytes (defends against memory exhaustion
32
44
  * from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
45
+ * Enforced by the engine for every transport: the built-in one aborts early, a
46
+ * custom one's body is checked when it arrives.
47
+ *
48
+ * Every numeric option must be a non-negative safe integer within its range;
49
+ * the constructor throws CkanValidationError otherwise (a negative or NaN
50
+ * timeout or cap would silently switch that guard off).
33
51
  */
34
52
  maxResponseBytes?: number;
35
53
  /** Injectable sleep, primarily for deterministic tests. */
36
54
  sleep?: (ms: number) => Promise<void>;
37
55
  }
38
- /** Most automatic retries the CLI's --max-retries accepts. */
56
+ /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
39
57
  export declare const MAX_RETRIES = 10;
58
+ /** Most redirects the engine follows (`maxRedirects`). */
59
+ export declare const MAX_REDIRECTS = 10;
60
+ /**
61
+ * Check a value for an HTTP header (headerValueProblem) and return it, or throw
62
+ * CkanValidationError `Invalid <name>: …`.
63
+ */
64
+ export declare function assertHeaderValue(name: string, value: string): string;
40
65
  /**
41
66
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
42
67
  * server asks for longer, the engine does not retry at all and surfaces the error at
43
- * once: retrying early would only land inside the window the server asked us to wait
44
- * out, and a hostile value must not stall the CLI.
68
+ * once, naming the requested wait: retrying early would only land inside the window
69
+ * the server asked us to wait out, and a hostile value must not stall the CLI.
45
70
  */
46
71
  export declare const MAX_RETRY_AFTER_MS = 30000;
47
72
  /**
@@ -82,8 +107,17 @@ export declare function isBidiControl(code: number): boolean;
82
107
  * Written as a char-code filter so no raw control byte appears in this source.
83
108
  */
84
109
  export declare function sanitizeServerText(text: string): string;
110
+ /**
111
+ * Check a base URL (baseUrlProblem: not blank, no whitespace or control
112
+ * characters, an absolute http(s) URL, no query string or fragment) and return it
113
+ * without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
114
+ * configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
115
+ * constructors call it, so a custom transport never sees a bad base URL; the
116
+ * default transport still re-checks the scheme on every hop.
117
+ */
118
+ export declare function validateBaseUrl(raw: string): string;
85
119
  export declare class RequestEngine {
86
- private readonly baseUrl;
120
+ #private;
87
121
  private readonly transport;
88
122
  private readonly userAgent;
89
123
  private readonly timeoutMs;
@@ -93,8 +127,36 @@ export declare class RequestEngine {
93
127
  private readonly maxResponseBytes;
94
128
  private readonly sleep;
95
129
  constructor(options?: EngineOptions);
96
- /** Build a fully-qualified URL from a path and optional query parameters. */
130
+ /**
131
+ * `text` without the base URL's credentials (raw and percent-decoded): server text
132
+ * (an error body or a CKAN error that echoes the request URL) and transport text
133
+ * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
134
+ * messages built from server text.
135
+ */
136
+ scrub(text: string): string;
137
+ /**
138
+ * A failure as the `cause` of the error the engine raises: the original when its
139
+ * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
140
+ * and the cause chain kept), so logging the error with its causes can't reveal the
141
+ * base URL's password.
142
+ */
143
+ private scrubCause;
144
+ /**
145
+ * Build a fully-qualified URL from a path and optional query parameters. It keeps
146
+ * the base URL's userinfo; `request()` leaves it out and sends it as an
147
+ * Authorization header instead (see basicAuthorization).
148
+ */
97
149
  buildUrl(path: string, query?: QueryParams): string;
150
+ /** buildUrl, with or without the base URL's userinfo. */
151
+ private composeUrl;
152
+ /**
153
+ * Call the transport under the overall deadline (`timeoutMs`): the request gets an
154
+ * AbortSignal that fires at the deadline, and the call rejects then whether the
155
+ * transport stops or not — a custom transport (fetch, a node:http wrapper) that
156
+ * ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
157
+ * becomes a rejection.
158
+ */
159
+ private callTransport;
98
160
  /** Perform a request with Accept negotiation and transient-error retries. */
99
161
  request(method: string, path: string, options?: {
100
162
  query?: QueryParams;
@@ -104,4 +166,3 @@ export declare class RequestEngine {
104
166
  getJson<T>(path: string, query?: QueryParams): Promise<T>;
105
167
  private toApiError;
106
168
  }
107
- //# sourceMappingURL=engine.d.ts.map
@@ -1,19 +1,42 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
- import { nodeHttpTransport } from "./http.js";
4
+ import { TextDecoder } from "node:util";
5
+ import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
5
6
  import { buildQueryString } from "./query.js";
6
- import { CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
7
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
8
+ import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
7
9
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
8
10
  const DEFAULT_USER_AGENT = "ckan-cli";
9
11
  const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
10
- /** Most automatic retries the CLI's --max-retries accepts. */
12
+ /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
11
13
  export const MAX_RETRIES = 10;
14
+ /** Most redirects the engine follows (`maxRedirects`). */
15
+ export const MAX_REDIRECTS = 10;
16
+ /**
17
+ * Check a value for an HTTP header (headerValueProblem) and return it, or throw
18
+ * CkanValidationError `Invalid <name>: …`.
19
+ */
20
+ export function assertHeaderValue(name, value) {
21
+ return assertValid(name, value, headerValueProblem);
22
+ }
23
+ /** Check an optional function option (`transport`, `sleep`): a function or undefined. */
24
+ function functionOption(name, value, fallback) {
25
+ if (value === undefined)
26
+ return fallback;
27
+ if (typeof value !== "function")
28
+ throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
29
+ return value;
30
+ }
31
+ /** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
32
+ function intOption(name, value, min, max) {
33
+ return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
34
+ }
12
35
  /**
13
36
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
14
37
  * server asks for longer, the engine does not retry at all and surfaces the error at
15
- * once: retrying early would only land inside the window the server asked us to wait
16
- * out, and a hostile value must not stall the CLI.
38
+ * once, naming the requested wait: retrying early would only land inside the window
39
+ * the server asked us to wait out, and a hostile value must not stall the CLI.
17
40
  */
18
41
  export const MAX_RETRY_AFTER_MS = 30_000;
19
42
  /** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
@@ -83,22 +106,15 @@ export function sanitizeServerText(text) {
83
106
  return out.replace(/\s+/g, " ").trim();
84
107
  }
85
108
  /**
86
- * Reject a base URL whose scheme is not http(s). The default transport already
87
- * gates this per hop, but the engine is exported as a library and may be handed a
88
- * custom transport that does no such check, so gate the configured base URL here
89
- * too (a `file:`/`ftp:` base URL fails fast with a typed error).
109
+ * Check a base URL (baseUrlProblem: not blank, no whitespace or control
110
+ * characters, an absolute http(s) URL, no query string or fragment) and return it
111
+ * without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
112
+ * configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
113
+ * constructors call it, so a custom transport never sees a bad base URL; the
114
+ * default transport still re-checks the scheme on every hop.
90
115
  */
91
- function assertHttpScheme(baseUrl) {
92
- let url;
93
- try {
94
- url = new URL(baseUrl);
95
- }
96
- catch {
97
- throw new CkanNetworkError(`Invalid base URL: ${redactUrl(baseUrl)}`);
98
- }
99
- if (url.protocol !== "http:" && url.protocol !== "https:") {
100
- throw new CkanNetworkError(`Unsupported protocol "${url.protocol}" in base URL: ${redactUrl(baseUrl)}`);
101
- }
116
+ export function validateBaseUrl(raw) {
117
+ return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
102
118
  }
103
119
  // The headers the engine sets itself, under the exact keys it uses. They are the
104
120
  // only ones that follow a cross-origin redirect.
@@ -112,9 +128,70 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
112
128
  function engineHeadersOnly(headers) {
113
129
  return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
114
130
  }
131
+ /** Why `value` is not a usable HttpResponse, or undefined when it is. */
132
+ function responseProblem(value) {
133
+ if (typeof value !== "object" || value === null)
134
+ return "not an object";
135
+ const r = value;
136
+ if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
137
+ return "status is not an HTTP status code";
138
+ }
139
+ if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
140
+ return "headers is not an object";
141
+ if (bodyBytes(r.body) === undefined)
142
+ return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
143
+ return undefined;
144
+ }
145
+ /**
146
+ * The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
147
+ * Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
148
+ * internal slot, not `instanceof`, so a value from another realm (a vm context, a
149
+ * Jest test) counts. Undefined for anything else.
150
+ */
151
+ function bodyBytes(value) {
152
+ if (Buffer.isBuffer(value))
153
+ return value;
154
+ if (ArrayBuffer.isView(value))
155
+ return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
156
+ const tag = Object.prototype.toString.call(value);
157
+ if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
158
+ return Buffer.from(value);
159
+ return undefined;
160
+ }
161
+ /**
162
+ * The response headers as a plain record with lower-case names. Node's transport
163
+ * lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
164
+ * and a fetch transport naturally returns its `Headers` object, which has no plain
165
+ * properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
166
+ * is copied.
167
+ */
168
+ function plainHeaders(headers) {
169
+ const h = headers;
170
+ if (typeof h.get === "function" && typeof h.forEach === "function") {
171
+ const record = {};
172
+ h.forEach.call(headers, (value, name) => {
173
+ record[String(name).toLowerCase()] = value;
174
+ });
175
+ return record;
176
+ }
177
+ const record = {};
178
+ for (const [name, value] of Object.entries(headers)) {
179
+ record[name.toLowerCase()] = value;
180
+ }
181
+ return record;
182
+ }
183
+ /** The first value of a header (a repeated one arrives as an array). */
184
+ function headerValue(value) {
185
+ return Array.isArray(value) ? value[0] : value;
186
+ }
115
187
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
116
188
  export class RequestEngine {
117
- baseUrl;
189
+ // A real private field (not TypeScript's `private`): util.inspect, console.log and
190
+ // JSON.stringify of a client never show it, so a password in the base URL can't be
191
+ // logged by accident. Messages use redactUrl.
192
+ #baseUrl;
193
+ /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
194
+ #credentials;
118
195
  transport;
119
196
  userAgent;
120
197
  timeoutMs;
@@ -124,75 +201,230 @@ export class RequestEngine {
124
201
  maxResponseBytes;
125
202
  sleep;
126
203
  constructor(options = {}) {
127
- this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
128
- // Re-check the base-URL scheme here, not only in the default transport: a
129
- // library consumer that injects a custom transport would otherwise get no
130
- // gating at all, and could be steered to a non-http(s) scheme.
131
- assertHttpScheme(this.baseUrl);
132
- this.transport = options.transport ?? nodeHttpTransport;
133
- this.userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
134
- this.timeoutMs = options.timeoutMs ?? 30_000;
135
- this.maxRetries = options.maxRetries ?? 2;
136
- this.retryDelayMs = options.retryDelayMs ?? 200;
137
- this.maxRedirects = options.maxRedirects ?? 5;
138
- this.maxResponseBytes = options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
139
- this.sleep = options.sleep ?? realSleep;
204
+ // The raw value, checked before the slash strip (the engine glues it into
205
+ // every URL), and here rather than only in the default transport: a library
206
+ // consumer that injects a custom transport would otherwise get no gating at
207
+ // all, and could be steered to a non-http(s) scheme. Only undefined selects
208
+ // the default.
209
+ this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
210
+ this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
211
+ try {
212
+ return [raw, decodeURIComponent(raw)];
213
+ }
214
+ catch {
215
+ return [raw];
216
+ }
217
+ });
218
+ this.transport = functionOption("transport", options.transport, nodeHttpTransport);
219
+ // Only undefined selects the default; a blank or unsendable value is refused.
220
+ this.userAgent =
221
+ options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
222
+ this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
223
+ this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
224
+ // Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
225
+ // after 1 ms, a burst rather than a backoff.
226
+ this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
227
+ this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
228
+ this.maxResponseBytes =
229
+ intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
230
+ this.sleep = functionOption("sleep", options.sleep, realSleep);
231
+ }
232
+ /**
233
+ * `text` without the base URL's credentials (raw and percent-decoded): server text
234
+ * (an error body or a CKAN error that echoes the request URL) and transport text
235
+ * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
236
+ * messages built from server text.
237
+ */
238
+ scrub(text) {
239
+ return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
140
240
  }
141
- /** Build a fully-qualified URL from a path and optional query parameters. */
241
+ /**
242
+ * A failure as the `cause` of the error the engine raises: the original when its
243
+ * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
244
+ * and the cause chain kept), so logging the error with its causes can't reveal the
245
+ * base URL's password.
246
+ */
247
+ scrubCause(cause, depth = 0) {
248
+ if (this.#credentials.length === 0 || depth > 5)
249
+ return cause;
250
+ if (typeof cause === "string")
251
+ return this.scrub(cause);
252
+ if (!(cause instanceof Error))
253
+ return cause;
254
+ const inner = this.scrubCause(cause.cause, depth + 1);
255
+ const message = this.scrub(cause.message);
256
+ if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
257
+ return cause;
258
+ }
259
+ const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
260
+ copy.name = cause.name;
261
+ const code = cause.code;
262
+ if (code !== undefined)
263
+ Object.assign(copy, { code });
264
+ return copy;
265
+ }
266
+ /**
267
+ * Build a fully-qualified URL from a path and optional query parameters. It keeps
268
+ * the base URL's userinfo; `request()` leaves it out and sends it as an
269
+ * Authorization header instead (see basicAuthorization).
270
+ */
142
271
  buildUrl(path, query) {
272
+ return this.composeUrl(path, query, true);
273
+ }
274
+ /** buildUrl, with or without the base URL's userinfo. */
275
+ composeUrl(path, query, withUserinfo) {
143
276
  const normalizedPath = path.startsWith("/") ? path : `/${path}`;
144
277
  const qs = query ? buildQueryString(query) : "";
145
- return `${this.baseUrl}${normalizedPath}${qs ? `?${qs}` : ""}`;
278
+ const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
279
+ return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
280
+ }
281
+ /**
282
+ * Call the transport under the overall deadline (`timeoutMs`): the request gets an
283
+ * AbortSignal that fires at the deadline, and the call rejects then whether the
284
+ * transport stops or not — a custom transport (fetch, a node:http wrapper) that
285
+ * ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
286
+ * becomes a rejection.
287
+ */
288
+ async callTransport(request) {
289
+ const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
290
+ if (this.timeoutMs === 0)
291
+ return call();
292
+ const controller = new AbortController();
293
+ let timer;
294
+ const deadline = new Promise((_, reject) => {
295
+ timer = setTimeout(() => {
296
+ const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
297
+ controller.abort(err);
298
+ reject(err);
299
+ }, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
300
+ });
301
+ try {
302
+ return await Promise.race([call(controller.signal), deadline]);
303
+ }
304
+ finally {
305
+ clearTimeout(timer);
306
+ }
146
307
  }
147
308
  /** Perform a request with Accept negotiation and transient-error retries. */
148
309
  async request(method, path, options = { accept: "application/json" }) {
149
- let url = this.buildUrl(path, options.query);
310
+ // The transport never sees the base URL's userinfo: the engine sends it as an
311
+ // Authorization header, per hop, so a redirect to the same origin (relative or
312
+ // absolute) keeps it and one to another origin or scheme drops it. A transport
313
+ // such as fetch also refuses a URL with credentials outright.
314
+ let url = this.composeUrl(path, options.query, false);
150
315
  let headers = {
151
316
  Accept: options.accept,
152
317
  "User-Agent": this.userAgent,
153
318
  };
319
+ const authorization = basicAuthorization(this.#baseUrl);
320
+ if (authorization !== undefined)
321
+ headers["Authorization"] = authorization;
322
+ /** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
323
+ let dropped;
154
324
  let attempt = 0;
155
325
  let redirects = 0;
156
326
  // attempts = initial try + maxRetries (redirects are counted separately)
157
327
  for (;;) {
158
- const response = await this.transport({
159
- method,
160
- url,
161
- headers,
162
- timeoutMs: this.timeoutMs,
163
- ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
164
- });
328
+ let response;
329
+ try {
330
+ response = await this.callTransport({
331
+ method,
332
+ url,
333
+ headers,
334
+ timeoutMs: this.timeoutMs,
335
+ redirect: "manual",
336
+ ...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
337
+ });
338
+ }
339
+ catch (cause) {
340
+ // The default transport rejects with CkanNetworkError only; an injected one may
341
+ // throw anything, and its text may quote the URL with its userinfo. Every
342
+ // failure becomes a CkanNetworkError naming the request (redacted), with the
343
+ // credentials scrubbed from its text and its cause chain; any other CkanError
344
+ // passes through.
345
+ if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
346
+ throw cause;
347
+ const reason = cause instanceof Error ? cause.message : String(cause);
348
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
349
+ cause: this.scrubCause(cause),
350
+ });
351
+ }
352
+ // An injected transport may resolve with anything; a malformed HttpResponse
353
+ // would otherwise surface below as a raw TypeError, outside the CkanError contract.
354
+ const invalid = responseProblem(response);
355
+ if (invalid !== undefined) {
356
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
357
+ }
358
+ // A transport must not follow redirects itself (`redirect: "manual"`): one that
359
+ // did (fetch's default) may have carried the Authorization header to another
360
+ // host, and the answer is not the one asked for. Reject it when it says so (`url`).
361
+ const finalUrl = response.url;
362
+ if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
363
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
364
+ `(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
365
+ `(HttpRequest.redirect is "manual").`);
366
+ }
165
367
  const status = response.status;
368
+ const responseHeaders = plainHeaders(response.headers);
369
+ // fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
370
+ const body = bodyBytes(response.body);
371
+ // The size cap holds whatever the transport did: the default one aborts early, a
372
+ // custom one may have read everything.
373
+ if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
374
+ throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
375
+ }
166
376
  const retryable = status === 429 || status === 503;
167
- if (retryable && attempt < this.maxRetries) {
168
- // Honour Retry-After; without a usable one, back off linearly. A Retry-After
169
- // beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
170
- const retryAfter = parseRetryAfter(response.headers["retry-after"]);
171
- if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
172
- attempt += 1;
173
- await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
174
- continue;
175
- }
377
+ // A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
378
+ // surfaces at once and names the wait the server asked for.
379
+ const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
380
+ const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
381
+ if (retryable && !tooLong && attempt < this.maxRetries) {
382
+ attempt += 1;
383
+ // Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
384
+ // for less: `Retry-After: 0` or a date in the past turned the retries into a
385
+ // zero-delay burst against a server that had just asked for less load.
386
+ const backoff = this.retryDelayMs * attempt;
387
+ await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
388
+ continue;
176
389
  }
177
390
  // Follow redirects, resolving the Location relative to the current URL.
178
- const location = response.headers["location"];
391
+ const location = headerValue(responseHeaders["location"]);
179
392
  const isRedirect = status >= 300 && status < 400;
180
393
  const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
181
394
  if (nextUrl !== undefined) {
395
+ // Userinfo in a Location is not used: credentials come from the base URL
396
+ // only, as the Authorization header, never from a server.
397
+ nextUrl.username = "";
398
+ nextUrl.password = "";
399
+ // Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
400
+ // never reaches a transport (a custom one may not check the scheme).
401
+ if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
402
+ throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
403
+ `for ${method} ${cutForMessage(redactUrl(url))}`);
404
+ }
182
405
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
183
406
  // port, so an https->http downgrade counts), keep only the engine's own
184
- // non-credential headers, so any future auth/cookie header is never
185
- // re-sent to a different host. The User-Agent stays: Hamburg's own
186
- // http: -> https: hop is cross-origin, and dropping it sent every request
187
- // through an http:// base URL without one (and ignored --user-agent).
188
- if (nextUrl.origin !== new URL(url).origin) {
407
+ // non-credential headers, so the base URL's Authorization (and any future
408
+ // auth/cookie header) is never re-sent to a different host. The same origin
409
+ // keeps it, whether the Location is relative or absolute. The User-Agent
410
+ // stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
411
+ // sent every request through an http:// base URL without one (and ignored
412
+ // --user-agent).
413
+ const from = new URL(url);
414
+ if (nextUrl.origin !== from.origin) {
415
+ if (headers["Authorization"] !== undefined && dropped === undefined) {
416
+ dropped =
417
+ from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
418
+ ? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
419
+ : `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
420
+ }
189
421
  headers = engineHeadersOnly(headers);
190
422
  }
191
423
  url = nextUrl.toString();
192
424
  redirects += 1;
193
425
  continue;
194
426
  }
195
- const contentType = String(response.headers["content-type"] ?? "");
427
+ const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
196
428
  if (status < 200 || status >= 300) {
197
429
  // A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
198
430
  // redirect this client cannot follow. Past the limit it is a loop; a
@@ -205,11 +437,14 @@ export class RequestEngine {
205
437
  : location
206
438
  ? `redirect to ${sanitizeServerText(location)} not followed`
207
439
  : "redirect not followed (no Location header)";
208
- throw new CkanApiError({ status, url, method, body: response.body.toString("utf8"), detail });
440
+ throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
209
441
  }
210
- throw this.toApiError(method, url, status, response.body);
442
+ throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
443
+ retries: attempt,
444
+ ...(tooLong ? { retryAfterMs: retryAfter } : {}),
445
+ });
211
446
  }
212
- return { data: response.body, contentType, status, url };
447
+ return { data: body, contentType, status, url };
213
448
  }
214
449
  }
215
450
  /** Perform a GET expecting JSON and parse it into `T`. */
@@ -226,13 +461,13 @@ export class RequestEngine {
226
461
  // 8-bit CSI U+009B): sanitise it before it reaches stderr.
227
462
  const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
228
463
  const message = /json/i.test(mediaType)
229
- ? `Invalid JSON from ${redactUrl(res.url)}`
230
- : `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
231
- throw new CkanParseError(message, { cause });
464
+ ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
465
+ : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
466
+ throw new CkanParseError(message, { cause: this.scrubCause(cause) });
232
467
  }
233
468
  }
234
- toApiError(method, url, status, body) {
235
- const text = body.toString("utf8");
469
+ toApiError(method, url, status, body, hint, retry) {
470
+ const text = this.scrub(body.toString("utf8"));
236
471
  let detail;
237
472
  try {
238
473
  const parsed = JSON.parse(text);
@@ -263,7 +498,43 @@ export class RequestEngine {
263
498
  // endpoint cannot inject terminal escape sequences via the stderr error message.
264
499
  if (detail !== undefined)
265
500
  detail = sanitizeServerText(detail);
266
- return new CkanApiError({ status, url, method, body: text, detail });
501
+ if (hint !== undefined)
502
+ detail = detail === undefined ? hint : `${detail}; ${hint}`;
503
+ return new CkanApiError({
504
+ status,
505
+ url,
506
+ method,
507
+ body: text,
508
+ detail,
509
+ retries: retry.retries,
510
+ ...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
511
+ });
512
+ }
513
+ }
514
+ /** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
515
+ function withoutUserinfo(url) {
516
+ const [userinfo] = credentialsIn(url);
517
+ return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
518
+ }
519
+ /**
520
+ * The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
521
+ * both percent-decoded, as Node's own http client builds it), or undefined without
522
+ * userinfo.
523
+ */
524
+ function basicAuthorization(url) {
525
+ const parsed = new URL(url);
526
+ if (parsed.username === "" && parsed.password === "")
527
+ return undefined;
528
+ const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
529
+ return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
530
+ }
531
+ /** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
532
+ function originOf(url) {
533
+ try {
534
+ return new URL(url).origin;
535
+ }
536
+ catch {
537
+ return url;
267
538
  }
268
539
  }
269
540
  /**
@@ -279,7 +550,7 @@ function decodeBody(body, contentType, url) {
279
550
  decoder = new TextDecoder(charset);
280
551
  }
281
552
  catch {
282
- throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
553
+ throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
283
554
  }
284
555
  return decoder.decode(body);
285
556
  }
@@ -294,4 +565,3 @@ function resolveLocation(location, base) {
294
565
  return undefined;
295
566
  }
296
567
  }
297
- //# sourceMappingURL=engine.js.map