@maschinenlesbar.org/ckan-cli 0.0.5 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +28 -14
  2. package/dist/src/cli/commands/catalogue.d.ts +1 -2
  3. package/dist/src/cli/commands/catalogue.js +22 -9
  4. package/dist/src/cli/commands/portal.d.ts +0 -1
  5. package/dist/src/cli/commands/portal.js +18 -10
  6. package/dist/src/cli/index.d.ts +0 -1
  7. package/dist/src/cli/index.js +2 -1
  8. package/dist/src/cli/io.d.ts +17 -1
  9. package/dist/src/cli/io.js +22 -1
  10. package/dist/src/cli/program.d.ts +0 -1
  11. package/dist/src/cli/program.js +24 -6
  12. package/dist/src/cli/run.d.ts +0 -1
  13. package/dist/src/cli/run.js +9 -2
  14. package/dist/src/cli/shared.d.ts +28 -14
  15. package/dist/src/cli/shared.js +78 -40
  16. package/dist/src/client/client.d.ts +13 -6
  17. package/dist/src/client/client.js +100 -35
  18. package/dist/src/client/engine.d.ts +79 -6
  19. package/dist/src/client/engine.js +188 -74
  20. package/dist/src/client/errors.d.ts +13 -1
  21. package/dist/src/client/errors.js +30 -3
  22. package/dist/src/client/http.d.ts +0 -1
  23. package/dist/src/client/http.js +44 -36
  24. package/dist/src/client/index.d.ts +6 -5
  25. package/dist/src/client/index.js +4 -4
  26. package/dist/src/client/portals-list.d.ts +0 -1
  27. package/dist/src/client/portals-list.js +0 -1
  28. package/dist/src/client/portals.d.ts +35 -2
  29. package/dist/src/client/portals.js +50 -3
  30. package/dist/src/client/query.d.ts +0 -1
  31. package/dist/src/client/query.js +0 -1
  32. package/dist/src/client/types.d.ts +5 -1
  33. package/dist/src/client/types.js +0 -1
  34. package/dist/src/client/validate.d.ts +55 -0
  35. package/dist/src/client/validate.js +119 -0
  36. package/dist/src/index.d.ts +0 -1
  37. package/dist/src/index.js +0 -1
  38. package/package.json +2 -1
  39. package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
  40. package/dist/src/cli/commands/catalogue.js.map +0 -1
  41. package/dist/src/cli/commands/portal.d.ts.map +0 -1
  42. package/dist/src/cli/commands/portal.js.map +0 -1
  43. package/dist/src/cli/index.d.ts.map +0 -1
  44. package/dist/src/cli/index.js.map +0 -1
  45. package/dist/src/cli/io.d.ts.map +0 -1
  46. package/dist/src/cli/io.js.map +0 -1
  47. package/dist/src/cli/program.d.ts.map +0 -1
  48. package/dist/src/cli/program.js.map +0 -1
  49. package/dist/src/cli/run.d.ts.map +0 -1
  50. package/dist/src/cli/run.js.map +0 -1
  51. package/dist/src/cli/shared.d.ts.map +0 -1
  52. package/dist/src/cli/shared.js.map +0 -1
  53. package/dist/src/client/client.d.ts.map +0 -1
  54. package/dist/src/client/client.js.map +0 -1
  55. package/dist/src/client/engine.d.ts.map +0 -1
  56. package/dist/src/client/engine.js.map +0 -1
  57. package/dist/src/client/errors.d.ts.map +0 -1
  58. package/dist/src/client/errors.js.map +0 -1
  59. package/dist/src/client/http.d.ts.map +0 -1
  60. package/dist/src/client/http.js.map +0 -1
  61. package/dist/src/client/index.d.ts.map +0 -1
  62. package/dist/src/client/index.js.map +0 -1
  63. package/dist/src/client/portals-list.d.ts.map +0 -1
  64. package/dist/src/client/portals-list.js.map +0 -1
  65. package/dist/src/client/portals.d.ts.map +0 -1
  66. package/dist/src/client/portals.js.map +0 -1
  67. package/dist/src/client/query.d.ts.map +0 -1
  68. package/dist/src/client/query.js.map +0 -1
  69. package/dist/src/client/types.d.ts.map +0 -1
  70. package/dist/src/client/types.js.map +0 -1
  71. package/dist/src/index.d.ts.map +0 -1
  72. package/dist/src/index.js.map +0 -1
@@ -1,49 +1,123 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
- import { nodeHttpTransport } from "./http.js";
4
+ import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  import { buildQueryString } from "./query.js";
6
- import { CkanApiError, CkanNetworkError, CkanParseError, describeCkanError } from "./errors.js";
6
+ import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
7
+ import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
7
8
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
8
9
  const DEFAULT_USER_AGENT = "ckan-cli";
9
10
  const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
11
+ /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
12
+ export const MAX_RETRIES = 10;
13
+ /** Most redirects the engine follows (`maxRedirects`). */
14
+ export const MAX_REDIRECTS = 10;
10
15
  /**
11
- * Strip control characters (all C0/C1 controls except tab and newline, plus DEL)
12
- * out of a string that originates in an attacker-controlled response — the error
13
- * `detail` extracted from the body. `JSON.parse` decodes an escaped ESC in an
14
- * error body into a real ESC byte, so without this a hostile/MITM'd endpoint could
15
- * drive ANSI/OSC escape sequences into the user's terminal when the message is
16
- * printed to stderr. The CLI's JSON output is escaped separately
17
- * (`escapeControlChars` in cli/shared.ts): `JSON.stringify` alone leaves DEL and
18
- * the C1 range raw. So this only needs to cover text that flows into an error message.
16
+ * Check a value for an HTTP header (headerValueProblem) and return it, or throw
17
+ * CkanValidationError `Invalid <name>: …`.
19
18
  */
20
- function sanitizeServerText(text) {
19
+ export function assertHeaderValue(name, value) {
20
+ return assertValid(name, value, headerValueProblem);
21
+ }
22
+ /** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
23
+ function intOption(name, value, min, max) {
24
+ return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
25
+ }
26
+ /**
27
+ * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
28
+ * server asks for longer, the engine does not retry at all and surfaces the error at
29
+ * once: retrying early would only land inside the window the server asked us to wait
30
+ * out, and a hostile value must not stall the CLI.
31
+ */
32
+ export const MAX_RETRY_AFTER_MS = 30_000;
33
+ /** An IMF-fixdate (RFC 9110 §5.6.7), the one HTTP-date form senders must generate. */
34
+ const IMF_FIXDATE = /^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/;
35
+ /**
36
+ * Parse a `Retry-After` header into a delay in milliseconds (RFC 9110 §10.2.3):
37
+ * either delay-seconds (`"120"`) or an HTTP-date (`"Wed, 21 Oct 2026 07:28:00 GMT"`,
38
+ * turned into the time left from `now`; a date in the past gives 0).
39
+ *
40
+ * Returns `undefined` when the header is absent or malformed — negative (`"-1"`),
41
+ * fractional (`"1.5"`), padded inside, any other date format — so the caller falls
42
+ * back to its own backoff. The strict patterns matter: `Date.parse` alone would
43
+ * read `"1.5"` as a date in 2001 and retry at once.
44
+ */
45
+ export function parseRetryAfter(header, now = Date.now()) {
46
+ const value = (Array.isArray(header) ? header[0] : header)?.trim();
47
+ if (value === undefined || value === "")
48
+ return undefined;
49
+ if (/^\d+$/.test(value))
50
+ return Number(value) * 1000;
51
+ if (!IMF_FIXDATE.test(value))
52
+ return undefined;
53
+ const when = Date.parse(value);
54
+ return Number.isNaN(when) ? undefined : Math.max(0, when - now);
55
+ }
56
+ /**
57
+ * True for the Unicode bidirectional formatting characters: ALM (U+061C), LRM/RLM
58
+ * (U+200E/U+200F), the embeddings and overrides U+202A–U+202E and the isolates
59
+ * U+2066–U+2069. A terminal applies them to the text that follows, so an override
60
+ * in server text can reorder what the user sees ("Trojan Source" spoofing).
61
+ */
62
+ export function isBidiControl(code) {
63
+ return (code === 0x061c ||
64
+ code === 0x200e ||
65
+ code === 0x200f ||
66
+ (code >= 0x202a && code <= 0x202e) ||
67
+ (code >= 0x2066 && code <= 0x2069));
68
+ }
69
+ /**
70
+ * Make a string that originates in an attacker-controlled response — the error
71
+ * `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
72
+ * into an error message on stderr:
73
+ *
74
+ * - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
75
+ * (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
76
+ * or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
77
+ * spoofing, title changes, OSC 52 clipboard writes).
78
+ * - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
79
+ * reorder the visible message.
80
+ * - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
81
+ * space and the ends are trimmed, so the text stays on one line and a server
82
+ * cannot forge an `Error:` line of its own.
83
+ *
84
+ * The CLI's JSON output is escaped separately (`escapeControlChars` in
85
+ * cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
86
+ * Written as a char-code filter so no raw control byte appears in this source.
87
+ */
88
+ export function sanitizeServerText(text) {
21
89
  let out = "";
22
90
  for (const ch of text) {
23
91
  const n = ch.codePointAt(0) ?? 0;
24
- if (n <= 8 || (n >= 0x0b && n <= 0x1f) || (n >= 0x7f && n <= 0x9f))
92
+ const whitespaceControl = n >= 0x09 && n <= 0x0d;
93
+ if (!whitespaceControl && (n <= 0x1f || (n >= 0x7f && n <= 0x9f) || isBidiControl(n)))
25
94
  continue;
26
95
  out += ch;
27
96
  }
28
- return out;
97
+ return out.replace(/\s+/g, " ").trim();
29
98
  }
30
99
  /**
31
- * Reject a base URL whose scheme is not http(s). The default transport already
32
- * gates this per hop, but the engine is exported as a library and may be handed a
33
- * custom transport that does no such check, so gate the configured base URL here
34
- * too (a `file:`/`ftp:` base URL fails fast with a typed error).
100
+ * Check a base URL (baseUrlProblem: not blank, no whitespace or control
101
+ * characters, an absolute http(s) URL, no query string or fragment) and return it
102
+ * without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
103
+ * configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
104
+ * constructors call it, so a custom transport never sees a bad base URL; the
105
+ * default transport still re-checks the scheme on every hop.
35
106
  */
36
- function assertHttpScheme(baseUrl) {
37
- let url;
38
- try {
39
- url = new URL(baseUrl);
40
- }
41
- catch {
42
- throw new CkanNetworkError(`Invalid base URL: ${baseUrl}`);
43
- }
44
- if (url.protocol !== "http:" && url.protocol !== "https:") {
45
- throw new CkanNetworkError(`Unsupported protocol "${url.protocol}" in base URL: ${baseUrl}`);
46
- }
107
+ export function validateBaseUrl(raw) {
108
+ return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
109
+ }
110
+ // The headers the engine sets itself, under the exact keys it uses. They are the
111
+ // only ones that follow a cross-origin redirect.
112
+ const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
113
+ /**
114
+ * A copy of `headers` with only the engine's own non-credential headers (used on
115
+ * cross-origin redirects). A list of known credential headers is never complete
116
+ * (Proxy-Authorization, X-Auth-Token, ...), so an allowlist is kept instead. A new
117
+ * object, so the one already handed to the transport is not changed.
118
+ */
119
+ function engineHeadersOnly(headers) {
120
+ return Object.fromEntries(Object.entries(headers).filter(([key]) => ENGINE_HEADERS.has(key)));
47
121
  }
48
122
  const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
49
123
  export class RequestEngine {
@@ -57,18 +131,22 @@ export class RequestEngine {
57
131
  maxResponseBytes;
58
132
  sleep;
59
133
  constructor(options = {}) {
60
- this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
61
- // Re-check the base-URL scheme here, not only in the default transport: a
62
- // library consumer that injects a custom transport would otherwise get no
63
- // gating at all, and could be steered to a non-http(s) scheme.
64
- assertHttpScheme(this.baseUrl);
134
+ // The raw value, checked before the slash strip (the engine glues it into
135
+ // every URL), and here rather than only in the default transport: a library
136
+ // consumer that injects a custom transport would otherwise get no gating at
137
+ // all, and could be steered to a non-http(s) scheme. Only undefined selects
138
+ // the default.
139
+ this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
65
140
  this.transport = options.transport ?? nodeHttpTransport;
66
- this.userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
67
- this.timeoutMs = options.timeoutMs ?? 30_000;
68
- this.maxRetries = options.maxRetries ?? 2;
69
- this.retryDelayMs = options.retryDelayMs ?? 200;
70
- this.maxRedirects = options.maxRedirects ?? 5;
71
- this.maxResponseBytes = options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
141
+ // Only undefined selects the default; a blank or unsendable value is refused.
142
+ this.userAgent =
143
+ options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
144
+ this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
145
+ this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
146
+ this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, Number.MAX_SAFE_INTEGER) ?? 200;
147
+ this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
148
+ this.maxResponseBytes =
149
+ intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
72
150
  this.sleep = options.sleep ?? realSleep;
73
151
  }
74
152
  /** Build a fully-qualified URL from a path and optional query parameters. */
@@ -98,42 +176,49 @@ export class RequestEngine {
98
176
  const status = response.status;
99
177
  const retryable = status === 429 || status === 503;
100
178
  if (retryable && attempt < this.maxRetries) {
101
- attempt += 1;
102
- await this.sleep(this.retryDelayMs * attempt);
103
- continue;
179
+ // Honour Retry-After; without a usable one, back off linearly. A Retry-After
180
+ // beyond MAX_RETRY_AFTER_MS is not retried: the error below surfaces at once.
181
+ const retryAfter = parseRetryAfter(response.headers["retry-after"]);
182
+ if (retryAfter === undefined || retryAfter <= MAX_RETRY_AFTER_MS) {
183
+ attempt += 1;
184
+ await this.sleep(retryAfter ?? this.retryDelayMs * attempt);
185
+ continue;
186
+ }
104
187
  }
105
188
  // Follow redirects, resolving the Location relative to the current URL.
106
- if (status >= 300 && status < 400 && redirects < this.maxRedirects) {
107
- const location = response.headers["location"];
108
- if (typeof location === "string" && location.length > 0) {
109
- const previous = new URL(url);
110
- const next = new URL(location, url);
111
- // Credential-strip guard: if the redirect crosses origin, drop the
112
- // request headers so any future auth/cookie header is never re-sent to
113
- // a different host. (Today only Accept/User-Agent are sent, but this
114
- // future-proofs against header leakage across origins.)
115
- if (next.origin !== previous.origin) {
116
- headers = { Accept: options.accept };
117
- }
118
- url = next.toString();
119
- redirects += 1;
120
- continue;
189
+ const location = response.headers["location"];
190
+ const isRedirect = status >= 300 && status < 400;
191
+ const nextUrl = isRedirect && redirects < this.maxRedirects ? resolveLocation(location, url) : undefined;
192
+ if (nextUrl !== undefined) {
193
+ // Credential-strip guard: if the redirect crosses origin (scheme + host +
194
+ // port, so an https->http downgrade counts), keep only the engine's own
195
+ // non-credential headers, so any future auth/cookie header is never
196
+ // re-sent to a different host. The User-Agent stays: Hamburg's own
197
+ // http: -> https: hop is cross-origin, and dropping it sent every request
198
+ // through an http:// base URL without one (and ignored --user-agent).
199
+ if (nextUrl.origin !== new URL(url).origin) {
200
+ headers = engineHeadersOnly(headers);
121
201
  }
202
+ url = nextUrl.toString();
203
+ redirects += 1;
204
+ continue;
122
205
  }
123
206
  const contentType = String(response.headers["content-type"] ?? "");
124
207
  if (status < 200 || status >= 300) {
125
- // A 3xx that was not followed because the limit was reached: say so, or a
126
- // bare "HTTP 301" reads like a redirect this client cannot follow.
127
- const exhausted = status >= 300 && status < 400 && Boolean(response.headers["location"]);
128
- throw exhausted
129
- ? new CkanApiError({
130
- status,
131
- url,
132
- method,
133
- body: response.body.toString("utf8"),
134
- detail: `stopped after ${this.maxRedirects} redirects (a redirect loop?)`,
135
- })
136
- : this.toApiError(method, url, status, response.body);
208
+ // A 3xx that was not followed: say why, or a bare "HTTP 301" reads like a
209
+ // redirect this client cannot follow. Past the limit it is a loop; a
210
+ // missing or malformed Location (which `new URL` would have thrown on as
211
+ // an "Unexpected error") is named as it came, sanitised.
212
+ if (isRedirect) {
213
+ const target = resolveLocation(location, url);
214
+ const detail = target !== undefined
215
+ ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
216
+ : location
217
+ ? `redirect to ${sanitizeServerText(location)} not followed`
218
+ : "redirect not followed (no Location header)";
219
+ throw new CkanApiError({ status, url, method, body: response.body.toString("utf8"), detail });
220
+ }
221
+ throw this.toApiError(method, url, status, response.body);
137
222
  }
138
223
  return { data: response.body, contentType, status, url };
139
224
  }
@@ -141,17 +226,19 @@ export class RequestEngine {
141
226
  /** Perform a GET expecting JSON and parse it into `T`. */
142
227
  async getJson(path, query) {
143
228
  const res = await this.request("GET", path, { query, accept: "application/json" });
144
- const text = res.data.toString("utf8");
229
+ const text = decodeBody(res.data, res.contentType, res.url);
145
230
  try {
146
231
  return JSON.parse(text);
147
232
  }
148
233
  catch (cause) {
149
234
  // Name the URL and what came back: a site that is not a CKAN (or not at this
150
235
  // path) typically answers with an HTML page and HTTP 200.
151
- const mediaType = res.contentType.split(";")[0].trim();
236
+ // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
237
+ // 8-bit CSI U+009B): sanitise it before it reaches stderr.
238
+ const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
152
239
  const message = /json/i.test(mediaType)
153
- ? `Invalid JSON from ${res.url}`
154
- : `Expected JSON from ${res.url} but got ${mediaType || "a body that is not JSON"}`;
240
+ ? `Invalid JSON from ${redactUrl(res.url)}`
241
+ : `Expected JSON from ${redactUrl(res.url)} but got ${mediaType || "a body that is not JSON"}`;
155
242
  throw new CkanParseError(message, { cause });
156
243
  }
157
244
  }
@@ -190,4 +277,31 @@ export class RequestEngine {
190
277
  return new CkanApiError({ status, url, method, body: text, detail });
191
278
  }
192
279
  }
193
- //# sourceMappingURL=engine.js.map
280
+ /**
281
+ * Decode a response body by the charset of its Content-Type (UTF-8 when none is
282
+ * given, as JSON requires). A leading byte-order mark is dropped: TextDecoder does
283
+ * that by default, where Buffer#toString kept it and JSON.parse then failed. CKAN
284
+ * sends UTF-8; this matters for proxies and mirrors that re-encode.
285
+ */
286
+ function decodeBody(body, contentType, url) {
287
+ const charset = /;\s*charset\s*=\s*"?([^";\s]+)"?/i.exec(contentType)?.[1] ?? "utf-8";
288
+ let decoder;
289
+ try {
290
+ decoder = new TextDecoder(charset);
291
+ }
292
+ catch {
293
+ throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${redactUrl(url)}.`);
294
+ }
295
+ return decoder.decode(body);
296
+ }
297
+ /** Resolve a Location header against the current URL; undefined if missing or malformed. */
298
+ function resolveLocation(location, base) {
299
+ if (location === undefined || location === "")
300
+ return undefined;
301
+ try {
302
+ return new URL(location, base);
303
+ }
304
+ catch {
305
+ return undefined;
306
+ }
307
+ }
@@ -1,3 +1,9 @@
1
+ /**
2
+ * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
3
+ * credential in a base URL never reaches an error message, a log or CI output.
4
+ * A URL without userinfo, or one that does not parse, is returned unchanged.
5
+ */
6
+ export declare function redactUrl(url: string): string;
1
7
  /** Base class for every error originating from this client. */
2
8
  export declare class CkanError extends Error {
3
9
  constructor(message: string, options?: {
@@ -30,6 +36,13 @@ export declare class CkanNetworkError extends CkanError {
30
36
  /** The response body could not be parsed as the expected JSON shape. */
31
37
  export declare class CkanParseError extends CkanError {
32
38
  }
39
+ /**
40
+ * An input the library refuses before sending any request: a client option or a
41
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
42
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
43
+ */
44
+ export declare class CkanValidationError extends CkanError {
45
+ }
33
46
  /**
34
47
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
35
48
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -37,4 +50,3 @@ export declare class CkanParseError extends CkanError {
37
50
  * (`{"__type": "Validation Error", "rows": ["Invalid integer"]}`).
38
51
  */
39
52
  export declare function describeCkanError(error: unknown): string;
40
- //# sourceMappingURL=errors.d.ts.map
@@ -1,5 +1,24 @@
1
1
  // Error types raised by the client. Kept free of any I/O so they are trivial to
2
2
  // construct in tests and to `instanceof`-check by consumers.
3
+ /**
4
+ * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
5
+ * credential in a base URL never reaches an error message, a log or CI output.
6
+ * A URL without userinfo, or one that does not parse, is returned unchanged.
7
+ */
8
+ export function redactUrl(url) {
9
+ let parsed;
10
+ try {
11
+ parsed = new URL(url);
12
+ }
13
+ catch {
14
+ return url;
15
+ }
16
+ if (parsed.username === "" && parsed.password === "")
17
+ return url;
18
+ parsed.username = "***";
19
+ parsed.password = "";
20
+ return parsed.href;
21
+ }
3
22
  /** Base class for every error originating from this client. */
4
23
  export class CkanError extends Error {
5
24
  constructor(message, options) {
@@ -18,10 +37,12 @@ export class CkanApiError extends CkanError {
18
37
  method;
19
38
  body;
20
39
  constructor(args) {
40
+ // The URL is shown without userinfo: a credential in --base-url must not leak.
41
+ const url = redactUrl(args.url);
21
42
  const detailPart = args.detail ? `: ${args.detail}` : "";
22
- super(`HTTP ${args.status} for ${args.method} ${args.url}${detailPart}`);
43
+ super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
23
44
  this.status = args.status;
24
- this.url = args.url;
45
+ this.url = url;
25
46
  this.method = args.method;
26
47
  this.body = args.body;
27
48
  this.detail = args.detail;
@@ -37,6 +58,13 @@ export class CkanNetworkError extends CkanError {
37
58
  /** The response body could not be parsed as the expected JSON shape. */
38
59
  export class CkanParseError extends CkanError {
39
60
  }
61
+ /**
62
+ * An input the library refuses before sending any request: a client option or a
63
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
64
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
65
+ */
66
+ export class CkanValidationError extends CkanError {
67
+ }
40
68
  /**
41
69
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
42
70
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -70,4 +98,3 @@ function solrReason(message) {
70
98
  return undefined;
71
99
  return match[1].replace(/\\+(['"])/g, "$1").trim();
72
100
  }
73
- //# sourceMappingURL=errors.js.map
@@ -28,4 +28,3 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
28
28
  * (connection errors, timeouts, malformed URLs).
29
29
  */
30
30
  export declare const nodeHttpTransport: Transport;
31
- //# sourceMappingURL=http.d.ts.map
@@ -7,7 +7,7 @@
7
7
  // `http.createServer` in the test-suite.
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
- import { CkanNetworkError } from "./errors.js";
10
+ import { CkanNetworkError, redactUrl } from "./errors.js";
11
11
  /**
12
12
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
13
13
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -24,14 +24,14 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
24
24
  url = new URL(request.url);
25
25
  }
26
26
  catch {
27
- reject(new CkanNetworkError(`Invalid URL: ${request.url}`));
27
+ reject(new CkanNetworkError(`Invalid URL: ${redactUrl(request.url)}`));
28
28
  return;
29
29
  }
30
30
  // Only http/https are supported. Reject anything else up front with a clear,
31
31
  // typed error instead of letting Node throw an opaque ERR_INVALID_PROTOCOL
32
32
  // (and so this never reaches the file:/ftp:/etc. drivers).
33
33
  if (url.protocol !== "http:" && url.protocol !== "https:") {
34
- reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${request.url}`));
34
+ reject(new CkanNetworkError(`Unsupported protocol "${url.protocol}" in URL: ${redactUrl(request.url)}`));
35
35
  return;
36
36
  }
37
37
  const isHttps = url.protocol === "https:";
@@ -50,40 +50,49 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
50
50
  clearTimeout(deadline);
51
51
  reject(err);
52
52
  };
53
- const req = driver.request(url, {
54
- method: request.method,
55
- headers: request.headers,
56
- }, (res) => {
57
- const chunks = [];
58
- let received = 0;
59
- let aborted = false;
60
- res.on("data", (chunk) => {
61
- if (aborted)
62
- return;
63
- received += chunk.length;
64
- if (maxBytes !== undefined && received > maxBytes) {
65
- aborted = true;
66
- res.destroy();
67
- settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
68
- return;
69
- }
70
- chunks.push(chunk);
71
- });
72
- res.on("end", () => {
73
- if (aborted)
74
- return;
75
- settleResolve({
76
- status: res.statusCode ?? 0,
77
- headers: res.headers,
78
- body: Buffer.concat(chunks),
53
+ // Node validates the headers synchronously and throws a bare TypeError
54
+ // (ERR_INVALID_CHAR) for one it cannot send; report it as a typed failure.
55
+ let req;
56
+ try {
57
+ req = driver.request(url, {
58
+ method: request.method,
59
+ headers: request.headers,
60
+ }, (res) => {
61
+ const chunks = [];
62
+ let received = 0;
63
+ let aborted = false;
64
+ res.on("data", (chunk) => {
65
+ if (aborted)
66
+ return;
67
+ received += chunk.length;
68
+ if (maxBytes !== undefined && received > maxBytes) {
69
+ aborted = true;
70
+ res.destroy();
71
+ settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
72
+ return;
73
+ }
74
+ chunks.push(chunk);
75
+ });
76
+ res.on("end", () => {
77
+ if (aborted)
78
+ return;
79
+ settleResolve({
80
+ status: res.statusCode ?? 0,
81
+ headers: res.headers,
82
+ body: Buffer.concat(chunks),
83
+ });
84
+ });
85
+ res.on("error", (err) => {
86
+ if (aborted)
87
+ return; // we already rejected with the size-cap error
88
+ settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
79
89
  });
80
90
  });
81
- res.on("error", (err) => {
82
- if (aborted)
83
- return; // we already rejected with the size-cap error
84
- settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
85
- });
86
- });
91
+ }
92
+ catch (err) {
93
+ reject(new CkanNetworkError(`Invalid request: ${err instanceof Error ? err.message : String(err)}`, { cause: err }));
94
+ return;
95
+ }
87
96
  if (request.timeoutMs && request.timeoutMs > 0) {
88
97
  // Two complementary guards, both using timeoutMs:
89
98
  // - setTimeout: a socket-*inactivity* timeout (no bytes for timeoutMs).
@@ -108,4 +117,3 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
108
117
  req.write(request.body);
109
118
  req.end();
110
119
  });
111
- //# sourceMappingURL=http.js.map
@@ -1,13 +1,14 @@
1
1
  export { CkanClient, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL } from "./engine.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
9
+ export { assertValid } from "./validate.js";
10
+ export type { Problem } from "./validate.js";
9
11
  export { PORTALS } from "./portals-list.js";
10
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
11
- export type { PortalCheck } from "./portals.js";
12
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
13
+ export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
12
14
  export * from "./types.js";
13
- //# sourceMappingURL=index.d.ts.map
@@ -1,10 +1,10 @@
1
1
  // Public entry point for the API client library.
2
2
  export { CkanClient, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL } from "./engine.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
7
+ export { assertValid } from "./validate.js";
7
8
  export { PORTALS } from "./portals-list.js";
8
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
9
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
9
10
  export * from "./types.js";
10
- //# sourceMappingURL=index.js.map
@@ -1,3 +1,2 @@
1
1
  import type { Portal } from "./types.js";
2
2
  export declare const PORTALS: readonly Portal[];
3
- //# sourceMappingURL=portals-list.d.ts.map
@@ -291,4 +291,3 @@ export const PORTALS = frozen([
291
291
  "note": "Research data of economics journals, not public-sector data."
292
292
  }
293
293
  ]);
294
- //# sourceMappingURL=portals-list.js.map
@@ -1,4 +1,5 @@
1
- import { type CkanClient } from "./client.js";
1
+ import { CkanClient } from "./client.js";
2
+ import type { EngineOptions } from "./engine.js";
2
3
  import type { Portal } from "./types.js";
3
4
  /**
4
5
  * One identity per portal, whatever form its URL was written in: host (without
@@ -34,4 +35,36 @@ export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
34
35
  export declare function withCheck(portal: Portal, check: PortalCheck, date: string): Portal;
35
36
  /** Run `fn` over `items` with at most `limit` calls in flight; results keep the input order. */
36
37
  export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
37
- //# sourceMappingURL=portals.d.ts.map
38
+ /** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
39
+ export declare const DEFAULT_CHECK_CONCURRENCY = 6;
40
+ /** Options for `checkPortalUrls` and `checkPortals`. */
41
+ export interface CheckPortalsOptions {
42
+ /** Client options for every check; `baseUrl` is replaced by each portal's URL. */
43
+ engineOptions?: EngineOptions;
44
+ /** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
45
+ createClient?: (options: EngineOptions) => CkanClient;
46
+ /** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
47
+ concurrency?: number;
48
+ /**
49
+ * When set, a failed check is repeated once after this pause (ms), so one
50
+ * timeout does not count as down. Unset (the default): a single try.
51
+ */
52
+ retryDelayMs?: number;
53
+ /** Injectable sleep for the retry pause, for tests. */
54
+ sleep?: (ms: number) => Promise<void>;
55
+ }
56
+ /**
57
+ * Check the CKAN portals at `urls` live (`checkPortal`), with at most
58
+ * `concurrency` checks in flight; results keep the input order. Never throws for
59
+ * one portal: a URL the client refuses becomes a failed check too. Rejects with
60
+ * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
61
+ */
62
+ export declare function checkPortalUrls(urls: readonly string[], options?: CheckPortalsOptions): Promise<PortalCheck[]>;
63
+ /**
64
+ * Check every portal in `portals` (the built-in list by default) live and fold
65
+ * each result into the entry (`withCheck`), dated `date` (default: today, UTC).
66
+ * What `ckan portals --check` prints.
67
+ */
68
+ export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
69
+ date?: string;
70
+ }): Promise<Portal[]>;