@maschinenlesbar.org/ckan-cli 0.2.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,9 @@
1
1
  // Shared helpers used across CLI command groups: option parsers, the global
2
2
  // option resolver, and the JSON result renderer.
3
3
  import { InvalidArgumentError } from "commander";
4
- import { isBidiControl } from "../client/engine.js";
5
- import { CkanError } from "../client/errors.js";
4
+ import { logOf } from "./io.js";
5
+ import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "../client/engine.js";
6
+ import { CkanError, cutForMessage, redactUrl } from "../client/errors.js";
6
7
  import { findPortal } from "../client/portals.js";
7
8
  import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
8
9
  import { PORTALS } from "../client/portals-list.js";
@@ -104,7 +105,7 @@ export function parseBaseUrl(value) {
104
105
  export function parsePortal(value) {
105
106
  const portal = findPortal(value, PORTALS);
106
107
  if (!portal) {
107
- throw new InvalidArgumentError(`Unknown portal "${value}". \`ckan portals\` lists the known ones.`);
108
+ throw new InvalidArgumentError(`Unknown portal "${cutForMessage(redactUrl(value))}". \`ckan portals\` lists the known ones.`);
108
109
  }
109
110
  return portal.url;
110
111
  }
@@ -169,6 +170,31 @@ export function renderJson(deps, global, value) {
169
170
  const text = escapeControlChars(stringifyJson(value, global.compact === true));
170
171
  deps.io.out(text);
171
172
  }
173
+ /**
174
+ * Log a warning (`ckan.http`) when the effective base URL (--portal >
175
+ * --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
176
+ * (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
177
+ * Called once per run, after the options are parsed and before the first request;
178
+ * stdout and the exit code are untouched.
179
+ */
180
+ export function warnOnCleartext(deps, global) {
181
+ const problem = cleartextProblem(global.portal ?? global.baseUrl ?? DEFAULT_BASE_URL);
182
+ if (problem !== undefined)
183
+ logOf(deps).warn("http", problem);
184
+ }
185
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
186
+ export function retryMessage(event) {
187
+ let host;
188
+ try {
189
+ host = new URL(event.url).host;
190
+ }
191
+ catch {
192
+ host = "the server";
193
+ }
194
+ const why = event.status === undefined ? "connection reset" : `HTTP ${event.status}`;
195
+ const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
196
+ return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
197
+ }
172
198
  /**
173
199
  * Wrap an async command action with consistent global-option resolution and
174
200
  * client construction. The callback receives a context (client + resolved global
@@ -182,7 +208,10 @@ export function action(deps, fn) {
182
208
  const command = args[args.length - 1];
183
209
  const positionals = args.slice(0, Math.max(0, args.length - 2));
184
210
  const global = command.optsWithGlobals();
185
- const client = deps.createClient(toEngineOptions(global));
211
+ warnOnCleartext(deps, global);
212
+ const options = toEngineOptions(global);
213
+ options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
214
+ const client = deps.createClient(options);
186
215
  await fn({ client, global, opts: command.opts() }, positionals);
187
216
  };
188
217
  }
@@ -1,7 +1,7 @@
1
1
  // CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
2
2
  // Action API (`<site>/api/3/action`).
3
3
  import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
- import { CkanError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
4
+ import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
5
5
  import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
6
6
  const ACTION = "/api/3/action";
7
7
  /**
@@ -159,12 +159,13 @@ export class CkanClient {
159
159
  // Another JSON API at the same path (or a proxy's JSON error page) is not an
160
160
  // envelope; say so rather than "failed: undefined".
161
161
  if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
162
- throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
162
+ throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
163
163
  }
164
164
  if (!env.success) {
165
165
  // A success:false envelope can come with HTTP 200, so it never passes the
166
166
  // engine's error-detail sanitising: strip terminal controls here too.
167
- throw new CkanError(`CKAN action "${name}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`);
167
+ const type = env.error?.__type;
168
+ throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
168
169
  }
169
170
  // `{"success": true}` without a result would print nothing useful (and the CLI
170
171
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
@@ -52,6 +52,24 @@ export interface EngineOptions {
52
52
  maxResponseBytes?: number;
53
53
  /** Injectable sleep, primarily for deterministic tests. */
54
54
  sleep?: (ms: number) => Promise<void>;
55
+ /**
56
+ * Called once per retry, right before the backoff sleep, for each retried 429/503
57
+ * and reset connection; never when there is no retry. A throw is swallowed.
58
+ */
59
+ onRetry?: (event: RetryEvent) => void;
60
+ }
61
+ /** What `EngineOptions.onRetry` is told about one retry. */
62
+ export interface RetryEvent {
63
+ /** Which retry this is, counting from 1. */
64
+ retry: number;
65
+ /** The most retries this request may make (`maxRetries`). */
66
+ maxRetries: number;
67
+ /** How long the engine waits before sending the request again. */
68
+ delayMs: number;
69
+ /** The HTTP status that caused the retry; absent for a reset connection. */
70
+ status?: number;
71
+ /** The URL being retried, userinfo redacted. */
72
+ url: string;
55
73
  }
56
74
  /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
57
75
  export declare const MAX_RETRIES = 10;
@@ -116,6 +134,19 @@ export declare function sanitizeServerText(text: string): string;
116
134
  * default transport still re-checks the scheme on every hop.
117
135
  */
118
136
  export declare function validateBaseUrl(raw: string): string;
137
+ /**
138
+ * Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
139
+ * or `undefined` when they would not: for `https:`, for a
140
+ * URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
141
+ * where nothing leaves the machine.
142
+ *
143
+ * The sentence names the host (`url.host`: host and port, never the userinfo) and what
144
+ * secret travels with the requests: the base URL's credentials when it carries userinfo,
145
+ * and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
146
+ * a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
147
+ * (once per run, before the first request).
148
+ */
149
+ export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
119
150
  export declare class RequestEngine {
120
151
  #private;
121
152
  private readonly transport;
@@ -126,9 +157,13 @@ export declare class RequestEngine {
126
157
  private readonly maxRedirects;
127
158
  private readonly maxResponseBytes;
128
159
  private readonly sleep;
160
+ private readonly onRetry;
129
161
  constructor(options?: EngineOptions);
162
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
163
+ private backOff;
130
164
  /**
131
- * `text` without the base URL's credentials (raw and percent-decoded): server text
165
+ * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
166
+ * server echoes them back in: `echoedCredentialForms`): server text
132
167
  * (an error body or a CKAN error that echoes the request URL) and transport text
133
168
  * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
134
169
  * messages built from server text.
@@ -4,7 +4,7 @@
4
4
  import { TextDecoder } from "node:util";
5
5
  import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
6
6
  import { buildQueryString } from "./query.js";
7
- import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
7
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
8
8
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
9
9
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
10
10
  const DEFAULT_USER_AGENT = "ckan-cli";
@@ -116,6 +116,39 @@ export function sanitizeServerText(text) {
116
116
  export function validateBaseUrl(raw) {
117
117
  return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
118
118
  }
119
+ /** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
120
+ function isLoopbackHost(hostname) {
121
+ return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
122
+ }
123
+ /**
124
+ * Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
125
+ * or `undefined` when they would not: for `https:`, for a
126
+ * URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
127
+ * where nothing leaves the machine.
128
+ *
129
+ * The sentence names the host (`url.host`: host and port, never the userinfo) and what
130
+ * secret travels with the requests: the base URL's credentials when it carries userinfo,
131
+ * and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
132
+ * a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
133
+ * (once per run, before the first request).
134
+ */
135
+ export function cleartextProblem(baseUrl, secrets = []) {
136
+ let url;
137
+ try {
138
+ url = new URL(baseUrl);
139
+ }
140
+ catch {
141
+ return undefined;
142
+ }
143
+ if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
144
+ return undefined;
145
+ const userinfo = url.username !== "" || url.password !== "";
146
+ const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
147
+ if (phrases.length === 0)
148
+ return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
149
+ const verb = phrases.length === 1 && !userinfo ? "is" : "are";
150
+ return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
151
+ }
119
152
  // The headers the engine sets itself, under the exact keys it uses. They are the
120
153
  // only ones that follow a cross-origin redirect.
121
154
  const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
@@ -192,6 +225,12 @@ export class RequestEngine {
192
225
  #baseUrl;
193
226
  /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
194
227
  #credentials;
228
+ /**
229
+ * The forms a server echoes that userinfo back in (the Basic value, the decoded
230
+ * `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
231
+ * password never leaves half of the `user:password` around it.
232
+ */
233
+ #echoed;
195
234
  transport;
196
235
  userAgent;
197
236
  timeoutMs;
@@ -200,6 +239,7 @@ export class RequestEngine {
200
239
  maxRedirects;
201
240
  maxResponseBytes;
202
241
  sleep;
242
+ onRetry;
203
243
  constructor(options = {}) {
204
244
  // The raw value, checked before the slash strip (the engine glues it into
205
245
  // every URL), and here rather than only in the default transport: a library
@@ -215,6 +255,9 @@ export class RequestEngine {
215
255
  return [raw];
216
256
  }
217
257
  });
258
+ this.#echoed = credentialsIn(this.#baseUrl)
259
+ .flatMap(echoedCredentialForms)
260
+ .sort((a, b) => b.length - a.length);
218
261
  this.transport = functionOption("transport", options.transport, nodeHttpTransport);
219
262
  // Only undefined selects the default; a blank or unsendable value is refused.
220
263
  this.userAgent =
@@ -228,15 +271,34 @@ export class RequestEngine {
228
271
  this.maxResponseBytes =
229
272
  intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
230
273
  this.sleep = functionOption("sleep", options.sleep, realSleep);
274
+ this.onRetry =
275
+ options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
276
+ }
277
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
278
+ async backOff(attempt, delayMs, url, status) {
279
+ try {
280
+ this.onRetry?.({
281
+ retry: attempt,
282
+ maxRetries: this.maxRetries,
283
+ delayMs,
284
+ ...(status !== undefined ? { status } : {}),
285
+ url: redactUrl(url),
286
+ });
287
+ }
288
+ catch {
289
+ // a logging hook is no reason to fail the request
290
+ }
291
+ await this.sleep(delayMs);
231
292
  }
232
293
  /**
233
- * `text` without the base URL's credentials (raw and percent-decoded): server text
294
+ * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
295
+ * server echoes them back in: `echoedCredentialForms`): server text
234
296
  * (an error body or a CKAN error that echoes the request URL) and transport text
235
297
  * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
236
298
  * messages built from server text.
237
299
  */
238
300
  scrub(text) {
239
- return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
301
+ return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
240
302
  }
241
303
  /**
242
304
  * A failure as the `cause` of the error the engine raises: the original when its
@@ -361,7 +423,7 @@ export class RequestEngine {
361
423
  const finalUrl = response.url;
362
424
  if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
363
425
  throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
364
- `(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
426
+ `(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
365
427
  `(HttpRequest.redirect is "manual").`);
366
428
  }
367
429
  const status = response.status;
@@ -384,7 +446,7 @@ export class RequestEngine {
384
446
  // for less: `Retry-After: 0` or a date in the past turned the retries into a
385
447
  // zero-delay burst against a server that had just asked for less load.
386
448
  const backoff = this.retryDelayMs * attempt;
387
- await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
449
+ await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
388
450
  continue;
389
451
  }
390
452
  // Follow redirects, resolving the Location relative to the current URL.
@@ -399,7 +461,7 @@ export class RequestEngine {
399
461
  // Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
400
462
  // never reaches a transport (a custom one may not check the scheme).
401
463
  if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
402
- throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
464
+ throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
403
465
  `for ${method} ${cutForMessage(redactUrl(url))}`);
404
466
  }
405
467
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
@@ -435,7 +497,7 @@ export class RequestEngine {
435
497
  const detail = target !== undefined
436
498
  ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
437
499
  : location
438
- ? `redirect to ${sanitizeServerText(location)} not followed`
500
+ ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
439
501
  : "redirect not followed (no Location header)";
440
502
  throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
441
503
  }
@@ -459,7 +521,7 @@ export class RequestEngine {
459
521
  // path) typically answers with an HTML page and HTTP 200.
460
522
  // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
461
523
  // 8-bit CSI U+009B): sanitise it before it reaches stderr.
462
- const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
524
+ const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
463
525
  const message = /json/i.test(mediaType)
464
526
  ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
465
527
  : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
@@ -550,7 +612,7 @@ function decodeBody(body, contentType, url) {
550
612
  decoder = new TextDecoder(charset);
551
613
  }
552
614
  catch {
553
- throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
615
+ throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
554
616
  }
555
617
  return decoder.decode(body);
556
618
  }
@@ -7,14 +7,26 @@
7
7
  */
8
8
  export declare function redactUrl(url: string): string;
9
9
  /**
10
- * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
11
- * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
12
- * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
13
- * is everything between `://` and the last `@` before the host. A value without a scheme
14
- * counts when it reads `user:password@host`. Used to redact those exact strings from text
10
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
11
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
12
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
13
+ * search text or a User-Agent as often as a credential, and the base URL always has a
14
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
15
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
15
16
  * that echoes the value (usage errors, help), whatever characters the password contains.
16
17
  */
17
18
  export declare function credentialsIn(value: string): string[];
19
+ /**
20
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
21
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
22
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
23
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
24
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
25
+ * are replaced as exact strings (`redactSecrets`).
26
+ */
27
+ export declare function echoedCredentialForms(userinfo: string): string[];
28
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
29
+ export declare function redactSecrets(text: string, secrets: readonly string[]): string;
18
30
  /**
19
31
  * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
20
32
  * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
@@ -24,11 +36,26 @@ export declare function redactCredentials(text: string, credentials: readonly st
24
36
  /**
25
37
  * Longest URL or server text (in characters) an error message shows. A long search
26
38
  * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
27
- * log. The error's `url`, `detail` and `body` properties keep the full value.
39
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
40
+ * value an own message quotes from a server answer or the user's input (a redirect
41
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
42
+ * this length too, so a library caller's `err.message` stays bounded.
28
43
  */
29
44
  export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
30
- /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
45
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
31
46
  export declare function cutForMessage(text: string): string;
47
+ /**
48
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
49
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
50
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
51
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
52
+ */
53
+ export declare function cutText(text: string, max: number): string;
54
+ /**
55
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
56
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
57
+ */
58
+ export declare function toWellFormed(text: string): string;
32
59
  /** Base class for every error originating from this client. */
33
60
  export declare class CkanError extends Error {
34
61
  constructor(message: string, options?: {
@@ -66,6 +93,23 @@ export declare class CkanApiError extends CkanError {
66
93
  /** True for statuses the API documents as transient and retry-able. */
67
94
  get isRetryable(): boolean;
68
95
  }
96
+ /**
97
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
98
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
99
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
100
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
101
+ * or undefined when the error carries none.
102
+ */
103
+ export declare class CkanActionError extends CkanError {
104
+ readonly action: string;
105
+ readonly errorType: string | undefined;
106
+ constructor(message: string, args: {
107
+ action: string;
108
+ errorType?: string | undefined;
109
+ });
110
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
111
+ get isNotFound(): boolean;
112
+ }
69
113
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
70
114
  export declare class CkanNetworkError extends CkanError {
71
115
  }
@@ -16,7 +16,8 @@ export function redactUrl(url) {
16
16
  // A value that doesn't parse can still carry credentials: cut them out by text.
17
17
  return redactCredentials(url, credentialsIn(url));
18
18
  }
19
- // `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
19
+ // A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
20
+ // the scheme "user:"), which is no URL with credentials at all.
20
21
  if (parsed.username === "" && parsed.password === "")
21
22
  return redactCredentials(url, credentialsIn(url));
22
23
  parsed.username = "***";
@@ -24,24 +25,22 @@ export function redactUrl(url) {
24
25
  return parsed.href;
25
26
  }
26
27
  /**
27
- * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
28
- * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
29
- * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
30
- * is everything between `://` and the last `@` before the host. A value without a scheme
31
- * counts when it reads `user:password@host`. Used to redact those exact strings from text
28
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
29
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
30
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
31
+ * search text or a User-Agent as often as a credential, and the base URL always has a
32
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
33
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
32
34
  * that echoes the value (usage errors, help), whatever characters the password contains.
33
35
  */
34
36
  export function credentialsIn(value) {
35
- const schemeAt = value.indexOf("://");
36
- const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
37
- // Without a scheme only the unmistakable `user:password@host` form counts.
38
- if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
37
+ const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
+ if (scheme === null)
39
39
  return [];
40
- // The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
41
- const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
40
+ const rest = value.slice(scheme[0].length);
42
41
  let parses = false;
43
42
  try {
44
- new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
43
+ new URL(value);
45
44
  parses = true;
46
45
  }
47
46
  catch {
@@ -53,6 +52,46 @@ export function credentialsIn(value) {
53
52
  const end = authority.lastIndexOf("@");
54
53
  return end > 0 ? [rest.slice(0, end)] : [];
55
54
  }
55
+ /**
56
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
57
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
58
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
59
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
60
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
61
+ * are replaced as exact strings (`redactSecrets`).
62
+ */
63
+ export function echoedCredentialForms(userinfo) {
64
+ const colon = userinfo.indexOf(":");
65
+ if (colon < 0)
66
+ return [];
67
+ const decode = (part) => {
68
+ try {
69
+ return decodeURIComponent(part);
70
+ }
71
+ catch {
72
+ return part;
73
+ }
74
+ };
75
+ const user = decode(userinfo.slice(0, colon));
76
+ const password = decode(userinfo.slice(colon + 1));
77
+ if (password === "")
78
+ return [];
79
+ const pair = `${user}:${password}`;
80
+ const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
81
+ if (password.length >= 4)
82
+ forms.push(password);
83
+ return forms;
84
+ }
85
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
86
+ export function redactSecrets(text, secrets) {
87
+ let out = text;
88
+ for (const secret of secrets) {
89
+ if (secret === "")
90
+ continue;
91
+ out = out.split(secret).join("***");
92
+ }
93
+ return out;
94
+ }
56
95
  /**
57
96
  * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
58
97
  * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
@@ -70,12 +109,37 @@ export function redactCredentials(text, credentials) {
70
109
  /**
71
110
  * Longest URL or server text (in characters) an error message shows. A long search
72
111
  * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
73
- * log. The error's `url`, `detail` and `body` properties keep the full value.
112
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
113
+ * value an own message quotes from a server answer or the user's input (a redirect
114
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
115
+ * this length too, so a library caller's `err.message` stays bounded.
74
116
  */
75
117
  export const MAX_MESSAGE_VALUE_LENGTH = 500;
76
- /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
118
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
77
119
  export function cutForMessage(text) {
78
- return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
120
+ return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
121
+ }
122
+ /**
123
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
124
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
125
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
126
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
127
+ */
128
+ export function cutText(text, max) {
129
+ if (text.length <= max)
130
+ return text;
131
+ const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
132
+ return text.slice(0, end);
133
+ }
134
+ function isHighSurrogate(c) {
135
+ return c >= 0xd800 && c <= 0xdbff;
136
+ }
137
+ /**
138
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
139
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
140
+ */
141
+ export function toWellFormed(text) {
142
+ return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
79
143
  }
80
144
  /** Base class for every error originating from this client. */
81
145
  export class CkanError extends Error {
@@ -134,6 +198,26 @@ export class CkanApiError extends CkanError {
134
198
  return this.status === 429 || this.status === 503;
135
199
  }
136
200
  }
201
+ /**
202
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
203
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
204
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
205
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
206
+ * or undefined when the error carries none.
207
+ */
208
+ export class CkanActionError extends CkanError {
209
+ action;
210
+ errorType;
211
+ constructor(message, args) {
212
+ super(message);
213
+ this.action = args.action;
214
+ this.errorType = args.errorType;
215
+ }
216
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
217
+ get isNotFound() {
218
+ return this.errorType === "Not Found Error";
219
+ }
220
+ }
137
221
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
138
222
  export class CkanNetworkError extends CkanError {
139
223
  }
@@ -1,11 +1,11 @@
1
1
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
- export type { EngineOptions, RawResponse } from "./engine.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
3
+ export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
9
  export { assertValid } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
@@ -1,9 +1,9 @@
1
1
  // Public entry point for the API client library.
2
2
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
7
  export { assertValid } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -2,7 +2,7 @@
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
3
  import { CkanClient, siteRoot } from "./client.js";
4
4
  import { validateBaseUrl } from "./engine.js";
5
- import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
5
+ import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
6
6
  import { PORTALS } from "./portals-list.js";
7
7
  import { assertValid, intRangeProblem } from "./validate.js";
8
8
  /**
@@ -118,9 +118,9 @@ function problemOf(err) {
118
118
  // The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
119
119
  const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
120
120
  if (reason !== undefined)
121
- return reason.slice(0, 80);
121
+ return cutText(reason, 80);
122
122
  }
123
- return message.slice(0, 80);
123
+ return cutText(message, 80);
124
124
  }
125
125
  /**
126
126
  * A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
@@ -2,7 +2,7 @@
2
2
  // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
3
  // enforce them with `assertValid` before any request; the CLI's value parsers call
4
4
  // the same functions, so a rule is written once and the CLI and the library agree.
5
- import { CkanValidationError } from "./errors.js";
5
+ import { CkanValidationError, cutForMessage } from "./errors.js";
6
6
  /**
7
7
  * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
8
  * something wrong with `value`; otherwise return `value` unchanged. A method that
@@ -147,7 +147,7 @@ export function baseUrlProblem(value) {
147
147
  return "Expected an absolute http(s) URL.";
148
148
  }
149
149
  if (url.protocol !== "http:" && url.protocol !== "https:") {
150
- return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
150
+ return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
151
151
  }
152
152
  if (/[?#]/.test(value))
153
153
  return "Expected a site URL without a query string or fragment.";