@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +23 -15
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +13 -12
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +0 -1
- package/dist/src/cli/io.d.ts +2 -2
- package/dist/src/cli/io.js +6 -3
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +17 -11
- package/dist/src/cli/run.d.ts +27 -1
- package/dist/src/cli/run.js +90 -2
- package/dist/src/cli/shared.d.ts +18 -15
- package/dist/src/cli/shared.js +45 -54
- package/dist/src/client/client.d.ts +30 -9
- package/dist/src/client/client.js +154 -37
- package/dist/src/client/engine.d.ts +74 -13
- package/dist/src/client/engine.js +341 -71
- package/dist/src/client/errors.d.ts +44 -2
- package/dist/src/client/errors.js +94 -6
- package/dist/src/client/http.d.ts +23 -1
- package/dist/src/client/http.js +54 -33
- package/dist/src/client/index.d.ts +7 -6
- package/dist/src/client/index.js +5 -5
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +86 -6
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +0 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +67 -0
- package/dist/src/client/validate.js +165 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +4 -3
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
|
@@ -13,35 +13,60 @@ export interface EngineOptions {
|
|
|
13
13
|
baseUrl?: string;
|
|
14
14
|
/** Swappable transport. Defaults to the built-in node http/https transport. */
|
|
15
15
|
transport?: Transport;
|
|
16
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* Value of the User-Agent header. Must be sendable (see `assertHeaderValue`):
|
|
18
|
+
* not blank, no control characters but tab, nothing above U+00FF.
|
|
19
|
+
*/
|
|
17
20
|
userAgent?: string;
|
|
18
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
|
|
23
|
+
* It covers the whole response, body included, and the engine enforces it for every
|
|
24
|
+
* transport (the request also gets an AbortSignal that fires at the deadline).
|
|
25
|
+
*/
|
|
19
26
|
timeoutMs?: number;
|
|
20
27
|
/**
|
|
21
|
-
* Number of automatic retries for transient (429/503) responses
|
|
22
|
-
*
|
|
23
|
-
*
|
|
28
|
+
* Number of automatic retries for transient (429/503) responses; a reset or other
|
|
29
|
+
* network error is not retried. Each waits `retryDelayMs * attempt`, or the
|
|
30
|
+
* response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
|
|
31
|
+
* one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
|
|
24
32
|
*/
|
|
25
33
|
maxRetries?: number;
|
|
26
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Base backoff between retries in milliseconds (grows linearly), 0 to
|
|
36
|
+
* `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
|
|
37
|
+
* `Retry-After`: the header can lengthen a wait, never shorten it.
|
|
38
|
+
*/
|
|
27
39
|
retryDelayMs?: number;
|
|
28
|
-
/** Number of HTTP redirects (301/302/303/307/308) to follow
|
|
40
|
+
/** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
|
|
29
41
|
maxRedirects?: number;
|
|
30
42
|
/**
|
|
31
43
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
32
44
|
* from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
|
|
45
|
+
* Enforced by the engine for every transport: the built-in one aborts early, a
|
|
46
|
+
* custom one's body is checked when it arrives.
|
|
47
|
+
*
|
|
48
|
+
* Every numeric option must be a non-negative safe integer within its range;
|
|
49
|
+
* the constructor throws CkanValidationError otherwise (a negative or NaN
|
|
50
|
+
* timeout or cap would silently switch that guard off).
|
|
33
51
|
*/
|
|
34
52
|
maxResponseBytes?: number;
|
|
35
53
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
36
54
|
sleep?: (ms: number) => Promise<void>;
|
|
37
55
|
}
|
|
38
|
-
/** Most automatic retries the CLI's --max-retries
|
|
56
|
+
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
39
57
|
export declare const MAX_RETRIES = 10;
|
|
58
|
+
/** Most redirects the engine follows (`maxRedirects`). */
|
|
59
|
+
export declare const MAX_REDIRECTS = 10;
|
|
60
|
+
/**
|
|
61
|
+
* Check a value for an HTTP header (headerValueProblem) and return it, or throw
|
|
62
|
+
* CkanValidationError `Invalid <name>: …`.
|
|
63
|
+
*/
|
|
64
|
+
export declare function assertHeaderValue(name: string, value: string): string;
|
|
40
65
|
/**
|
|
41
66
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
42
67
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
43
|
-
* once: retrying early would only land inside the window
|
|
44
|
-
* out, and a hostile value must not stall the CLI.
|
|
68
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
69
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
45
70
|
*/
|
|
46
71
|
export declare const MAX_RETRY_AFTER_MS = 30000;
|
|
47
72
|
/**
|
|
@@ -82,8 +107,17 @@ export declare function isBidiControl(code: number): boolean;
|
|
|
82
107
|
* Written as a char-code filter so no raw control byte appears in this source.
|
|
83
108
|
*/
|
|
84
109
|
export declare function sanitizeServerText(text: string): string;
|
|
110
|
+
/**
|
|
111
|
+
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
112
|
+
* characters, an absolute http(s) URL, no query string or fragment) and return it
|
|
113
|
+
* without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
|
|
114
|
+
* configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
|
|
115
|
+
* constructors call it, so a custom transport never sees a bad base URL; the
|
|
116
|
+
* default transport still re-checks the scheme on every hop.
|
|
117
|
+
*/
|
|
118
|
+
export declare function validateBaseUrl(raw: string): string;
|
|
85
119
|
export declare class RequestEngine {
|
|
86
|
-
private
|
|
120
|
+
#private;
|
|
87
121
|
private readonly transport;
|
|
88
122
|
private readonly userAgent;
|
|
89
123
|
private readonly timeoutMs;
|
|
@@ -93,8 +127,36 @@ export declare class RequestEngine {
|
|
|
93
127
|
private readonly maxResponseBytes;
|
|
94
128
|
private readonly sleep;
|
|
95
129
|
constructor(options?: EngineOptions);
|
|
96
|
-
/**
|
|
130
|
+
/**
|
|
131
|
+
* `text` without the base URL's credentials (raw and percent-decoded): server text
|
|
132
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
133
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
134
|
+
* messages built from server text.
|
|
135
|
+
*/
|
|
136
|
+
scrub(text: string): string;
|
|
137
|
+
/**
|
|
138
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
139
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
140
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
141
|
+
* base URL's password.
|
|
142
|
+
*/
|
|
143
|
+
private scrubCause;
|
|
144
|
+
/**
|
|
145
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
146
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
147
|
+
* Authorization header instead (see basicAuthorization).
|
|
148
|
+
*/
|
|
97
149
|
buildUrl(path: string, query?: QueryParams): string;
|
|
150
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
151
|
+
private composeUrl;
|
|
152
|
+
/**
|
|
153
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
154
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
155
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
156
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
157
|
+
* becomes a rejection.
|
|
158
|
+
*/
|
|
159
|
+
private callTransport;
|
|
98
160
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
99
161
|
request(method: string, path: string, options?: {
|
|
100
162
|
query?: QueryParams;
|
|
@@ -104,4 +166,3 @@ export declare class RequestEngine {
|
|
|
104
166
|
getJson<T>(path: string, query?: QueryParams): Promise<T>;
|
|
105
167
|
private toApiError;
|
|
106
168
|
}
|
|
107
|
-
//# sourceMappingURL=engine.d.ts.map
|
|
@@ -1,19 +1,42 @@
|
|
|
1
1
|
// The request engine: turns logical (method, path, query) calls into HTTP
|
|
2
2
|
// requests via a Transport, applies retry/backoff for transient statuses
|
|
3
3
|
// (429, 503), and decodes responses.
|
|
4
|
-
import {
|
|
4
|
+
import { TextDecoder } from "node:util";
|
|
5
|
+
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
5
6
|
import { buildQueryString } from "./query.js";
|
|
6
|
-
import { CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
|
|
7
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
|
|
8
|
+
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
7
9
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
8
10
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
9
11
|
const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
|
|
10
|
-
/** Most automatic retries the CLI's --max-retries
|
|
12
|
+
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
11
13
|
export const MAX_RETRIES = 10;
|
|
14
|
+
/** Most redirects the engine follows (`maxRedirects`). */
|
|
15
|
+
export const MAX_REDIRECTS = 10;
|
|
16
|
+
/**
|
|
17
|
+
* Check a value for an HTTP header (headerValueProblem) and return it, or throw
|
|
18
|
+
* CkanValidationError `Invalid <name>: …`.
|
|
19
|
+
*/
|
|
20
|
+
export function assertHeaderValue(name, value) {
|
|
21
|
+
return assertValid(name, value, headerValueProblem);
|
|
22
|
+
}
|
|
23
|
+
/** Check an optional function option (`transport`, `sleep`): a function or undefined. */
|
|
24
|
+
function functionOption(name, value, fallback) {
|
|
25
|
+
if (value === undefined)
|
|
26
|
+
return fallback;
|
|
27
|
+
if (typeof value !== "function")
|
|
28
|
+
throw new CkanValidationError(`Invalid ${name}: Expected a function.`);
|
|
29
|
+
return value;
|
|
30
|
+
}
|
|
31
|
+
/** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
|
|
32
|
+
function intOption(name, value, min, max) {
|
|
33
|
+
return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
|
|
34
|
+
}
|
|
12
35
|
/**
|
|
13
36
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
14
37
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
15
|
-
* once: retrying early would only land inside the window
|
|
16
|
-
* out, and a hostile value must not stall the CLI.
|
|
38
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
39
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
17
40
|
*/
|
|
18
41
|
export const MAX_RETRY_AFTER_MS = 30_000;
|
|
19
42
|
/** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
|
|
@@ -83,22 +106,15 @@ export function sanitizeServerText(text) {
|
|
|
83
106
|
return out.replace(/\s+/g, " ").trim();
|
|
84
107
|
}
|
|
85
108
|
/**
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
109
|
+
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
110
|
+
* characters, an absolute http(s) URL, no query string or fragment) and return it
|
|
111
|
+
* without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
|
|
112
|
+
* configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
|
|
113
|
+
* constructors call it, so a custom transport never sees a bad base URL; the
|
|
114
|
+
* default transport still re-checks the scheme on every hop.
|
|
90
115
|
*/
|
|
91
|
-
function
|
|
92
|
-
|
|
93
|
-
try {
|
|
94
|
-
url = new URL(baseUrl);
|
|
95
|
-
}
|
|
96
|
-
catch {
|
|
97
|
-
throw new CkanNetworkError(`Invalid base URL: ${redactUrl(baseUrl)}`);
|
|
98
|
-
}
|
|
99
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
100
|
-
throw new CkanNetworkError(`Unsupported protocol "${url.protocol}" in base URL: ${redactUrl(baseUrl)}`);
|
|
101
|
-
}
|
|
116
|
+
export function validateBaseUrl(raw) {
|
|
117
|
+
return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
102
118
|
}
|
|
103
119
|
// The headers the engine sets itself, under the exact keys it uses. They are the
|
|
104
120
|
// only ones that follow a cross-origin redirect.
|
|
@@ -112,9 +128,70 @@ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
|
112
128
|
function engineHeadersOnly(headers) {
|
|
113
129
|
return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
|
|
114
130
|
}
|
|
131
|
+
/** Why `value` is not a usable HttpResponse, or undefined when it is. */
|
|
132
|
+
function responseProblem(value) {
|
|
133
|
+
if (typeof value !== "object" || value === null)
|
|
134
|
+
return "not an object";
|
|
135
|
+
const r = value;
|
|
136
|
+
if (typeof r.status !== "number" || !Number.isInteger(r.status) || r.status < 100 || r.status > 599) {
|
|
137
|
+
return "status is not an HTTP status code";
|
|
138
|
+
}
|
|
139
|
+
if (typeof r.headers !== "object" || r.headers === null || Array.isArray(r.headers))
|
|
140
|
+
return "headers is not an object";
|
|
141
|
+
if (bodyBytes(r.body) === undefined)
|
|
142
|
+
return "body is not a Buffer, Uint8Array, other ArrayBuffer view or ArrayBuffer";
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* The response body as a Buffer (a view, no copy): a Buffer, any ArrayBuffer view (a
|
|
147
|
+
* Uint8Array from fetch, a DataView) or an ArrayBuffer/SharedArrayBuffer — checked by
|
|
148
|
+
* internal slot, not `instanceof`, so a value from another realm (a vm context, a
|
|
149
|
+
* Jest test) counts. Undefined for anything else.
|
|
150
|
+
*/
|
|
151
|
+
function bodyBytes(value) {
|
|
152
|
+
if (Buffer.isBuffer(value))
|
|
153
|
+
return value;
|
|
154
|
+
if (ArrayBuffer.isView(value))
|
|
155
|
+
return Buffer.from(value.buffer, value.byteOffset, value.byteLength);
|
|
156
|
+
const tag = Object.prototype.toString.call(value);
|
|
157
|
+
if (tag === "[object ArrayBuffer]" || tag === "[object SharedArrayBuffer]")
|
|
158
|
+
return Buffer.from(value);
|
|
159
|
+
return undefined;
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* The response headers as a plain record with lower-case names. Node's transport
|
|
163
|
+
* lower-cases them; a custom one may not (`Retry-After`, `Location`, `Content-Type`),
|
|
164
|
+
* and a fetch transport naturally returns its `Headers` object, which has no plain
|
|
165
|
+
* properties. Such an object (anything with `get` and `forEach`: `Headers`, a `Map`)
|
|
166
|
+
* is copied.
|
|
167
|
+
*/
|
|
168
|
+
function plainHeaders(headers) {
|
|
169
|
+
const h = headers;
|
|
170
|
+
if (typeof h.get === "function" && typeof h.forEach === "function") {
|
|
171
|
+
const record = {};
|
|
172
|
+
h.forEach.call(headers, (value, name) => {
|
|
173
|
+
record[String(name).toLowerCase()] = value;
|
|
174
|
+
});
|
|
175
|
+
return record;
|
|
176
|
+
}
|
|
177
|
+
const record = {};
|
|
178
|
+
for (const [name, value] of Object.entries(headers)) {
|
|
179
|
+
record[name.toLowerCase()] = value;
|
|
180
|
+
}
|
|
181
|
+
return record;
|
|
182
|
+
}
|
|
183
|
+
/** The first value of a header (a repeated one arrives as an array). */
|
|
184
|
+
function headerValue(value) {
|
|
185
|
+
return Array.isArray(value) ? value[0] : value;
|
|
186
|
+
}
|
|
115
187
|
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
116
188
|
export class RequestEngine {
|
|
117
|
-
|
|
189
|
+
// A real private field (not TypeScript's `private`): util.inspect, console.log and
|
|
190
|
+
// JSON.stringify of a client never show it, so a password in the base URL can't be
|
|
191
|
+
// logged by accident. Messages use redactUrl.
|
|
192
|
+
#baseUrl;
|
|
193
|
+
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
194
|
+
#credentials;
|
|
118
195
|
transport;
|
|
119
196
|
userAgent;
|
|
120
197
|
timeoutMs;
|
|
@@ -124,75 +201,230 @@ export class RequestEngine {
|
|
|
124
201
|
maxResponseBytes;
|
|
125
202
|
sleep;
|
|
126
203
|
constructor(options = {}) {
|
|
127
|
-
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
|
|
132
|
-
this
|
|
133
|
-
this
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
204
|
+
// The raw value, checked before the slash strip (the engine glues it into
|
|
205
|
+
// every URL), and here rather than only in the default transport: a library
|
|
206
|
+
// consumer that injects a custom transport would otherwise get no gating at
|
|
207
|
+
// all, and could be steered to a non-http(s) scheme. Only undefined selects
|
|
208
|
+
// the default.
|
|
209
|
+
this.#baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
210
|
+
this.#credentials = credentialsIn(this.#baseUrl).flatMap((raw) => {
|
|
211
|
+
try {
|
|
212
|
+
return [raw, decodeURIComponent(raw)];
|
|
213
|
+
}
|
|
214
|
+
catch {
|
|
215
|
+
return [raw];
|
|
216
|
+
}
|
|
217
|
+
});
|
|
218
|
+
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
219
|
+
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
220
|
+
this.userAgent =
|
|
221
|
+
options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
|
|
222
|
+
this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
|
|
223
|
+
this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
|
|
224
|
+
// Bounded like a Retry-After wait: a larger value overflowed Node's timer and fired
|
|
225
|
+
// after 1 ms, a burst rather than a backoff.
|
|
226
|
+
this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, MAX_RETRY_AFTER_MS) ?? 200;
|
|
227
|
+
this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
|
|
228
|
+
this.maxResponseBytes =
|
|
229
|
+
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
230
|
+
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
231
|
+
}
|
|
232
|
+
/**
|
|
233
|
+
* `text` without the base URL's credentials (raw and percent-decoded): server text
|
|
234
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
235
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
236
|
+
* messages built from server text.
|
|
237
|
+
*/
|
|
238
|
+
scrub(text) {
|
|
239
|
+
return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
|
|
140
240
|
}
|
|
141
|
-
/**
|
|
241
|
+
/**
|
|
242
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
243
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
244
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
245
|
+
* base URL's password.
|
|
246
|
+
*/
|
|
247
|
+
scrubCause(cause, depth = 0) {
|
|
248
|
+
if (this.#credentials.length === 0 || depth > 5)
|
|
249
|
+
return cause;
|
|
250
|
+
if (typeof cause === "string")
|
|
251
|
+
return this.scrub(cause);
|
|
252
|
+
if (!(cause instanceof Error))
|
|
253
|
+
return cause;
|
|
254
|
+
const inner = this.scrubCause(cause.cause, depth + 1);
|
|
255
|
+
const message = this.scrub(cause.message);
|
|
256
|
+
if (message === cause.message && inner === cause.cause && this.scrub(cause.stack ?? "") === (cause.stack ?? "")) {
|
|
257
|
+
return cause;
|
|
258
|
+
}
|
|
259
|
+
const copy = new Error(message, inner === undefined ? undefined : { cause: inner });
|
|
260
|
+
copy.name = cause.name;
|
|
261
|
+
const code = cause.code;
|
|
262
|
+
if (code !== undefined)
|
|
263
|
+
Object.assign(copy, { code });
|
|
264
|
+
return copy;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
268
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
269
|
+
* Authorization header instead (see basicAuthorization).
|
|
270
|
+
*/
|
|
142
271
|
buildUrl(path, query) {
|
|
272
|
+
return this.composeUrl(path, query, true);
|
|
273
|
+
}
|
|
274
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
275
|
+
composeUrl(path, query, withUserinfo) {
|
|
143
276
|
const normalizedPath = path.startsWith("/") ? path : `/${path}`;
|
|
144
277
|
const qs = query ? buildQueryString(query) : "";
|
|
145
|
-
|
|
278
|
+
const base = withUserinfo ? this.#baseUrl : withoutUserinfo(this.#baseUrl);
|
|
279
|
+
return `${base}${normalizedPath}${qs ? `?${qs}` : ""}`;
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
283
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
284
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
285
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
286
|
+
* becomes a rejection.
|
|
287
|
+
*/
|
|
288
|
+
async callTransport(request) {
|
|
289
|
+
const call = (signal) => Promise.resolve().then(() => this.transport(signal === undefined ? request : { ...request, signal }));
|
|
290
|
+
if (this.timeoutMs === 0)
|
|
291
|
+
return call();
|
|
292
|
+
const controller = new AbortController();
|
|
293
|
+
let timer;
|
|
294
|
+
const deadline = new Promise((_, reject) => {
|
|
295
|
+
timer = setTimeout(() => {
|
|
296
|
+
const err = new CkanNetworkError(`Request exceeded the ${this.timeoutMs}ms deadline`);
|
|
297
|
+
controller.abort(err);
|
|
298
|
+
reject(err);
|
|
299
|
+
}, Math.min(this.timeoutMs, MAX_TIMEOUT_MS));
|
|
300
|
+
});
|
|
301
|
+
try {
|
|
302
|
+
return await Promise.race([call(controller.signal), deadline]);
|
|
303
|
+
}
|
|
304
|
+
finally {
|
|
305
|
+
clearTimeout(timer);
|
|
306
|
+
}
|
|
146
307
|
}
|
|
147
308
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
148
309
|
async request(method, path, options = { accept: "application/json" }) {
|
|
149
|
-
|
|
310
|
+
// The transport never sees the base URL's userinfo: the engine sends it as an
|
|
311
|
+
// Authorization header, per hop, so a redirect to the same origin (relative or
|
|
312
|
+
// absolute) keeps it and one to another origin or scheme drops it. A transport
|
|
313
|
+
// such as fetch also refuses a URL with credentials outright.
|
|
314
|
+
let url = this.composeUrl(path, options.query, false);
|
|
150
315
|
let headers = {
|
|
151
316
|
Accept: options.accept,
|
|
152
317
|
"User-Agent": this.userAgent,
|
|
153
318
|
};
|
|
319
|
+
const authorization = basicAuthorization(this.#baseUrl);
|
|
320
|
+
if (authorization !== undefined)
|
|
321
|
+
headers["Authorization"] = authorization;
|
|
322
|
+
/** Why a redirect dropped the base URL's credentials, for a 401/403 message. */
|
|
323
|
+
let dropped;
|
|
154
324
|
let attempt = 0;
|
|
155
325
|
let redirects = 0;
|
|
156
326
|
// attempts = initial try + maxRetries (redirects are counted separately)
|
|
157
327
|
for (;;) {
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
328
|
+
let response;
|
|
329
|
+
try {
|
|
330
|
+
response = await this.callTransport({
|
|
331
|
+
method,
|
|
332
|
+
url,
|
|
333
|
+
headers,
|
|
334
|
+
timeoutMs: this.timeoutMs,
|
|
335
|
+
redirect: "manual",
|
|
336
|
+
...(this.maxResponseBytes > 0 ? { maxResponseBytes: this.maxResponseBytes } : {}),
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
catch (cause) {
|
|
340
|
+
// The default transport rejects with CkanNetworkError only; an injected one may
|
|
341
|
+
// throw anything, and its text may quote the URL with its userinfo. Every
|
|
342
|
+
// failure becomes a CkanNetworkError naming the request (redacted), with the
|
|
343
|
+
// credentials scrubbed from its text and its cause chain; any other CkanError
|
|
344
|
+
// passes through.
|
|
345
|
+
if (cause instanceof CkanError && !(cause instanceof CkanNetworkError))
|
|
346
|
+
throw cause;
|
|
347
|
+
const reason = cause instanceof Error ? cause.message : String(cause);
|
|
348
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sanitizeServerText(this.scrub(reason))}`, {
|
|
349
|
+
cause: this.scrubCause(cause),
|
|
350
|
+
});
|
|
351
|
+
}
|
|
352
|
+
// An injected transport may resolve with anything; a malformed HttpResponse
|
|
353
|
+
// would otherwise surface below as a raw TypeError, outside the CkanError contract.
|
|
354
|
+
const invalid = responseProblem(response);
|
|
355
|
+
if (invalid !== undefined) {
|
|
356
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport returned an invalid response (${invalid}).`);
|
|
357
|
+
}
|
|
358
|
+
// A transport must not follow redirects itself (`redirect: "manual"`): one that
|
|
359
|
+
// did (fetch's default) may have carried the Authorization header to another
|
|
360
|
+
// host, and the answer is not the one asked for. Reject it when it says so (`url`).
|
|
361
|
+
const finalUrl = response.url;
|
|
362
|
+
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
363
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
|
|
364
|
+
`(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
|
|
365
|
+
`(HttpRequest.redirect is "manual").`);
|
|
366
|
+
}
|
|
165
367
|
const status = response.status;
|
|
368
|
+
const responseHeaders = plainHeaders(response.headers);
|
|
369
|
+
// fetch gives a Uint8Array; view it as a Buffer (no copy), which the decoders expect.
|
|
370
|
+
const body = bodyBytes(response.body);
|
|
371
|
+
// The size cap holds whatever the transport did: the default one aborts early, a
|
|
372
|
+
// custom one may have read everything.
|
|
373
|
+
if (this.maxResponseBytes > 0 && body.byteLength > this.maxResponseBytes) {
|
|
374
|
+
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: ${sizeLimitMessage(this.maxResponseBytes)}`);
|
|
375
|
+
}
|
|
166
376
|
const retryable = status === 429 || status === 503;
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
377
|
+
// A Retry-After beyond MAX_RETRY_AFTER_MS is not retried: the error below
|
|
378
|
+
// surfaces at once and names the wait the server asked for.
|
|
379
|
+
const retryAfter = retryable ? parseRetryAfter(headerValue(responseHeaders["retry-after"])) : undefined;
|
|
380
|
+
const tooLong = retryAfter !== undefined && retryAfter > MAX_RETRY_AFTER_MS;
|
|
381
|
+
if (retryable && !tooLong && attempt < this.maxRetries) {
|
|
382
|
+
attempt += 1;
|
|
383
|
+
// Back off linearly from retryDelayMs. A Retry-After can ask for longer, never
|
|
384
|
+
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
385
|
+
// zero-delay burst against a server that had just asked for less load.
|
|
386
|
+
const backoff = this.retryDelayMs * attempt;
|
|
387
|
+
await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
|
|
388
|
+
continue;
|
|
176
389
|
}
|
|
177
390
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
178
|
-
const location =
|
|
391
|
+
const location = headerValue(responseHeaders["location"]);
|
|
179
392
|
const isRedirect = status >= 300 && status < 400;
|
|
180
393
|
const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
|
|
181
394
|
if (nextUrl !== undefined) {
|
|
395
|
+
// Userinfo in a Location is not used: credentials come from the base URL
|
|
396
|
+
// only, as the Authorization header, never from a server.
|
|
397
|
+
nextUrl.username = "";
|
|
398
|
+
nextUrl.password = "";
|
|
399
|
+
// Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
|
|
400
|
+
// never reaches a transport (a custom one may not check the scheme).
|
|
401
|
+
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
402
|
+
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
|
|
403
|
+
`for ${method} ${cutForMessage(redactUrl(url))}`);
|
|
404
|
+
}
|
|
182
405
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
183
406
|
// port, so an https->http downgrade counts), keep only the engine's own
|
|
184
|
-
// non-credential headers, so
|
|
185
|
-
// re-sent to a different host. The
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
407
|
+
// non-credential headers, so the base URL's Authorization (and any future
|
|
408
|
+
// auth/cookie header) is never re-sent to a different host. The same origin
|
|
409
|
+
// keeps it, whether the Location is relative or absolute. The User-Agent
|
|
410
|
+
// stays: Hamburg's own http: -> https: hop is cross-origin, and dropping it
|
|
411
|
+
// sent every request through an http:// base URL without one (and ignored
|
|
412
|
+
// --user-agent).
|
|
413
|
+
const from = new URL(url);
|
|
414
|
+
if (nextUrl.origin !== from.origin) {
|
|
415
|
+
if (headers["Authorization"] !== undefined && dropped === undefined) {
|
|
416
|
+
dropped =
|
|
417
|
+
from.protocol === "http:" && nextUrl.protocol === "https:" && from.hostname === nextUrl.hostname
|
|
418
|
+
? "the server redirected http→https, which dropped the base URL's credentials; use an https base URL"
|
|
419
|
+
: `the redirect to ${nextUrl.origin} dropped the base URL's credentials (they are sent to their own origin only)`;
|
|
420
|
+
}
|
|
189
421
|
headers = engineHeadersOnly(headers);
|
|
190
422
|
}
|
|
191
423
|
url = nextUrl.toString();
|
|
192
424
|
redirects += 1;
|
|
193
425
|
continue;
|
|
194
426
|
}
|
|
195
|
-
const contentType = String(
|
|
427
|
+
const contentType = String(headerValue(responseHeaders["content-type"]) ?? "");
|
|
196
428
|
if (status < 200 || status >= 300) {
|
|
197
429
|
// A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
|
|
198
430
|
// redirect this client cannot follow. Past the limit it is a loop; a
|
|
@@ -205,11 +437,14 @@ export class RequestEngine {
|
|
|
205
437
|
: location
|
|
206
438
|
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
207
439
|
: "redirect not followed (no Location header)";
|
|
208
|
-
throw new CkanApiError({ status, url, method, body:
|
|
440
|
+
throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
|
|
209
441
|
}
|
|
210
|
-
throw this.toApiError(method, url, status,
|
|
442
|
+
throw this.toApiError(method, url, status, body, status === 401 || status === 403 ? dropped : undefined, {
|
|
443
|
+
retries: attempt,
|
|
444
|
+
...(tooLong ? { retryAfterMs: retryAfter } : {}),
|
|
445
|
+
});
|
|
211
446
|
}
|
|
212
|
-
return { data:
|
|
447
|
+
return { data: body, contentType, status, url };
|
|
213
448
|
}
|
|
214
449
|
}
|
|
215
450
|
/** Perform a GET expecting JSON and parse it into `T`. */
|
|
@@ -226,13 +461,13 @@ export class RequestEngine {
|
|
|
226
461
|
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
227
462
|
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
228
463
|
const message = /json/i.test(mediaType)
|
|
229
|
-
? `Invalid JSON from ${redactUrl(res.url)}`
|
|
230
|
-
: `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
|
|
231
|
-
throw new CkanParseError(message, { cause });
|
|
464
|
+
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
465
|
+
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
466
|
+
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
232
467
|
}
|
|
233
468
|
}
|
|
234
|
-
toApiError(method, url, status, body) {
|
|
235
|
-
const text = body.toString("utf8");
|
|
469
|
+
toApiError(method, url, status, body, hint, retry) {
|
|
470
|
+
const text = this.scrub(body.toString("utf8"));
|
|
236
471
|
let detail;
|
|
237
472
|
try {
|
|
238
473
|
const parsed = JSON.parse(text);
|
|
@@ -263,7 +498,43 @@ export class RequestEngine {
|
|
|
263
498
|
// endpoint cannot inject terminal escape sequences via the stderr error message.
|
|
264
499
|
if (detail !== undefined)
|
|
265
500
|
detail = sanitizeServerText(detail);
|
|
266
|
-
|
|
501
|
+
if (hint !== undefined)
|
|
502
|
+
detail = detail === undefined ? hint : `${detail}; ${hint}`;
|
|
503
|
+
return new CkanApiError({
|
|
504
|
+
status,
|
|
505
|
+
url,
|
|
506
|
+
method,
|
|
507
|
+
body: text,
|
|
508
|
+
detail,
|
|
509
|
+
retries: retry.retries,
|
|
510
|
+
...(retry.retryAfterMs === undefined ? {} : { retryAfterMs: retry.retryAfterMs, maxRetryAfterMs: MAX_RETRY_AFTER_MS }),
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
/** `url` without its userinfo (`https://user:pw@host/x` → `https://host/x`), as written otherwise. */
|
|
515
|
+
function withoutUserinfo(url) {
|
|
516
|
+
const [userinfo] = credentialsIn(url);
|
|
517
|
+
return userinfo === undefined ? url : url.replace(`://${userinfo}@`, "://");
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* The `Authorization` header for a URL's userinfo (`Basic base64(user:password)`,
|
|
521
|
+
* both percent-decoded, as Node's own http client builds it), or undefined without
|
|
522
|
+
* userinfo.
|
|
523
|
+
*/
|
|
524
|
+
function basicAuthorization(url) {
|
|
525
|
+
const parsed = new URL(url);
|
|
526
|
+
if (parsed.username === "" && parsed.password === "")
|
|
527
|
+
return undefined;
|
|
528
|
+
const pair = `${decodeURIComponent(parsed.username)}:${decodeURIComponent(parsed.password)}`;
|
|
529
|
+
return `Basic ${Buffer.from(pair, "utf8").toString("base64")}`;
|
|
530
|
+
}
|
|
531
|
+
/** The origin (scheme, host, port) of a URL, or the value itself if it doesn't parse. */
|
|
532
|
+
function originOf(url) {
|
|
533
|
+
try {
|
|
534
|
+
return new URL(url).origin;
|
|
535
|
+
}
|
|
536
|
+
catch {
|
|
537
|
+
return url;
|
|
267
538
|
}
|
|
268
539
|
}
|
|
269
540
|
/**
|
|
@@ -279,7 +550,7 @@ function decodeBody(body, contentType, url) {
|
|
|
279
550
|
decoder = new TextDecoder(charset);
|
|
280
551
|
}
|
|
281
552
|
catch {
|
|
282
|
-
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
|
|
553
|
+
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
|
|
283
554
|
}
|
|
284
555
|
return decoder.decode(body);
|
|
285
556
|
}
|
|
@@ -294,4 +565,3 @@ function resolveLocation(location, base) {
|
|
|
294
565
|
return undefined;
|
|
295
566
|
}
|
|
296
567
|
}
|
|
297
|
-
//# sourceMappingURL=engine.js.map
|