@maschinenlesbar.org/ckan-cli 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -4,7 +4,7 @@
4
4
  import { TextDecoder } from "node:util";
5
5
  import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
6
6
  import { buildQueryString } from "./query.js";
7
- import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
7
+ import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
8
8
  import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
9
9
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
10
10
  const DEFAULT_USER_AGENT = "ckan-cli";
@@ -116,6 +116,39 @@ export function sanitizeServerText(text) {
116
116
  export function validateBaseUrl(raw) {
117
117
  return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
118
118
  }
119
+ /** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
120
+ function isLoopbackHost(hostname) {
121
+ return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
122
+ }
123
+ /**
124
+ * Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
125
+ * or `undefined` when they would not: for `https:`, for a
126
+ * URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
127
+ * where nothing leaves the machine.
128
+ *
129
+ * The sentence names the host (`url.host`: host and port, never the userinfo) and what
130
+ * secret travels with the requests: the base URL's credentials when it carries userinfo,
131
+ * and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
132
+ * a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
133
+ * (once per run, before the first request).
134
+ */
135
+ export function cleartextProblem(baseUrl, secrets = []) {
136
+ let url;
137
+ try {
138
+ url = new URL(baseUrl);
139
+ }
140
+ catch {
141
+ return undefined;
142
+ }
143
+ if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
144
+ return undefined;
145
+ const userinfo = url.username !== "" || url.password !== "";
146
+ const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
147
+ if (phrases.length === 0)
148
+ return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
149
+ const verb = phrases.length === 1 && !userinfo ? "is" : "are";
150
+ return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
151
+ }
119
152
  // The headers the engine sets itself, under the exact keys it uses. They are the
120
153
  // only ones that follow a cross-origin redirect.
121
154
  const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
@@ -192,6 +225,12 @@ export class RequestEngine {
192
225
  #baseUrl;
193
226
  /** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
194
227
  #credentials;
228
+ /**
229
+ * The forms a server echoes that userinfo back in (the Basic value, the decoded
230
+ * `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
231
+ * password never leaves half of the `user:password` around it.
232
+ */
233
+ #echoed;
195
234
  transport;
196
235
  userAgent;
197
236
  timeoutMs;
@@ -215,6 +254,9 @@ export class RequestEngine {
215
254
  return [raw];
216
255
  }
217
256
  });
257
+ this.#echoed = credentialsIn(this.#baseUrl)
258
+ .flatMap(echoedCredentialForms)
259
+ .sort((a, b) => b.length - a.length);
218
260
  this.transport = functionOption("transport", options.transport, nodeHttpTransport);
219
261
  // Only undefined selects the default; a blank or unsendable value is refused.
220
262
  this.userAgent =
@@ -230,13 +272,14 @@ export class RequestEngine {
230
272
  this.sleep = functionOption("sleep", options.sleep, realSleep);
231
273
  }
232
274
  /**
233
- * `text` without the base URL's credentials (raw and percent-decoded): server text
275
+ * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
276
+ * server echoes them back in: `echoedCredentialForms`): server text
234
277
  * (an error body or a CKAN error that echoes the request URL) and transport text
235
278
  * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
236
279
  * messages built from server text.
237
280
  */
238
281
  scrub(text) {
239
- return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
282
+ return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
240
283
  }
241
284
  /**
242
285
  * A failure as the `cause` of the error the engine raises: the original when its
@@ -361,7 +404,7 @@ export class RequestEngine {
361
404
  const finalUrl = response.url;
362
405
  if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
363
406
  throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
364
- `(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
407
+ `(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
365
408
  `(HttpRequest.redirect is "manual").`);
366
409
  }
367
410
  const status = response.status;
@@ -399,7 +442,7 @@ export class RequestEngine {
399
442
  // Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
400
443
  // never reaches a transport (a custom one may not check the scheme).
401
444
  if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
402
- throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
445
+ throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
403
446
  `for ${method} ${cutForMessage(redactUrl(url))}`);
404
447
  }
405
448
  // Credential-strip guard: if the redirect crosses origin (scheme + host +
@@ -435,7 +478,7 @@ export class RequestEngine {
435
478
  const detail = target !== undefined
436
479
  ? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
437
480
  : location
438
- ? `redirect to ${sanitizeServerText(location)} not followed`
481
+ ? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
439
482
  : "redirect not followed (no Location header)";
440
483
  throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
441
484
  }
@@ -459,7 +502,7 @@ export class RequestEngine {
459
502
  // path) typically answers with an HTML page and HTTP 200.
460
503
  // The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
461
504
  // 8-bit CSI U+009B): sanitise it before it reaches stderr.
462
- const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
505
+ const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
463
506
  const message = /json/i.test(mediaType)
464
507
  ? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
465
508
  : `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
@@ -550,7 +593,7 @@ function decodeBody(body, contentType, url) {
550
593
  decoder = new TextDecoder(charset);
551
594
  }
552
595
  catch {
553
- throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
596
+ throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
554
597
  }
555
598
  return decoder.decode(body);
556
599
  }
@@ -7,14 +7,26 @@
7
7
  */
8
8
  export declare function redactUrl(url: string): string;
9
9
  /**
10
- * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
11
- * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
12
- * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
13
- * is everything between `://` and the last `@` before the host. A value without a scheme
14
- * counts when it reads `user:password@host`. Used to redact those exact strings from text
10
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
11
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
12
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
13
+ * search text or a User-Agent as often as a credential, and the base URL always has a
14
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
15
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
15
16
  * that echoes the value (usage errors, help), whatever characters the password contains.
16
17
  */
17
18
  export declare function credentialsIn(value: string): string[];
19
+ /**
20
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
21
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
22
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
23
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
24
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
25
+ * are replaced as exact strings (`redactSecrets`).
26
+ */
27
+ export declare function echoedCredentialForms(userinfo: string): string[];
28
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
29
+ export declare function redactSecrets(text: string, secrets: readonly string[]): string;
18
30
  /**
19
31
  * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
20
32
  * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
@@ -24,11 +36,26 @@ export declare function redactCredentials(text: string, credentials: readonly st
24
36
  /**
25
37
  * Longest URL or server text (in characters) an error message shows. A long search
26
38
  * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
27
- * log. The error's `url`, `detail` and `body` properties keep the full value.
39
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
40
+ * value an own message quotes from a server answer or the user's input (a redirect
41
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
42
+ * this length too, so a library caller's `err.message` stays bounded.
28
43
  */
29
44
  export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
30
- /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
45
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
31
46
  export declare function cutForMessage(text: string): string;
47
+ /**
48
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
49
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
50
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
51
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
52
+ */
53
+ export declare function cutText(text: string, max: number): string;
54
+ /**
55
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
56
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
57
+ */
58
+ export declare function toWellFormed(text: string): string;
32
59
  /** Base class for every error originating from this client. */
33
60
  export declare class CkanError extends Error {
34
61
  constructor(message: string, options?: {
@@ -66,6 +93,23 @@ export declare class CkanApiError extends CkanError {
66
93
  /** True for statuses the API documents as transient and retry-able. */
67
94
  get isRetryable(): boolean;
68
95
  }
96
+ /**
97
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
98
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
99
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
100
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
101
+ * or undefined when the error carries none.
102
+ */
103
+ export declare class CkanActionError extends CkanError {
104
+ readonly action: string;
105
+ readonly errorType: string | undefined;
106
+ constructor(message: string, args: {
107
+ action: string;
108
+ errorType?: string | undefined;
109
+ });
110
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
111
+ get isNotFound(): boolean;
112
+ }
69
113
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
70
114
  export declare class CkanNetworkError extends CkanError {
71
115
  }
@@ -16,7 +16,8 @@ export function redactUrl(url) {
16
16
  // A value that doesn't parse can still carry credentials: cut them out by text.
17
17
  return redactCredentials(url, credentialsIn(url));
18
18
  }
19
- // `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
19
+ // A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
20
+ // the scheme "user:"), which is no URL with credentials at all.
20
21
  if (parsed.username === "" && parsed.password === "")
21
22
  return redactCredentials(url, credentialsIn(url));
22
23
  parsed.username = "***";
@@ -24,24 +25,22 @@ export function redactUrl(url) {
24
25
  return parsed.href;
25
26
  }
26
27
  /**
27
- * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
28
- * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
29
- * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
30
- * is everything between `://` and the last `@` before the host. A value without a scheme
31
- * counts when it reads `user:password@host`. Used to redact those exact strings from text
28
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
29
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
30
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
31
+ * search text or a User-Agent as often as a credential, and the base URL always has a
32
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
33
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
32
34
  * that echoes the value (usage errors, help), whatever characters the password contains.
33
35
  */
34
36
  export function credentialsIn(value) {
35
- const schemeAt = value.indexOf("://");
36
- const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
37
- // Without a scheme only the unmistakable `user:password@host` form counts.
38
- if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
37
+ const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
+ if (scheme === null)
39
39
  return [];
40
- // The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
41
- const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
40
+ const rest = value.slice(scheme[0].length);
42
41
  let parses = false;
43
42
  try {
44
- new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
43
+ new URL(value);
45
44
  parses = true;
46
45
  }
47
46
  catch {
@@ -53,6 +52,46 @@ export function credentialsIn(value) {
53
52
  const end = authority.lastIndexOf("@");
54
53
  return end > 0 ? [rest.slice(0, end)] : [];
55
54
  }
55
+ /**
56
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
57
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
58
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
59
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
60
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
61
+ * are replaced as exact strings (`redactSecrets`).
62
+ */
63
+ export function echoedCredentialForms(userinfo) {
64
+ const colon = userinfo.indexOf(":");
65
+ if (colon < 0)
66
+ return [];
67
+ const decode = (part) => {
68
+ try {
69
+ return decodeURIComponent(part);
70
+ }
71
+ catch {
72
+ return part;
73
+ }
74
+ };
75
+ const user = decode(userinfo.slice(0, colon));
76
+ const password = decode(userinfo.slice(colon + 1));
77
+ if (password === "")
78
+ return [];
79
+ const pair = `${user}:${password}`;
80
+ const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
81
+ if (password.length >= 4)
82
+ forms.push(password);
83
+ return forms;
84
+ }
85
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
86
+ export function redactSecrets(text, secrets) {
87
+ let out = text;
88
+ for (const secret of secrets) {
89
+ if (secret === "")
90
+ continue;
91
+ out = out.split(secret).join("***");
92
+ }
93
+ return out;
94
+ }
56
95
  /**
57
96
  * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
58
97
  * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
@@ -70,12 +109,37 @@ export function redactCredentials(text, credentials) {
70
109
  /**
71
110
  * Longest URL or server text (in characters) an error message shows. A long search
72
111
  * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
73
- * log. The error's `url`, `detail` and `body` properties keep the full value.
112
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
113
+ * value an own message quotes from a server answer or the user's input (a redirect
114
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
115
+ * this length too, so a library caller's `err.message` stays bounded.
74
116
  */
75
117
  export const MAX_MESSAGE_VALUE_LENGTH = 500;
76
- /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
118
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
77
119
  export function cutForMessage(text) {
78
- return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
120
+ return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
121
+ }
122
+ /**
123
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
124
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
125
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
126
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
127
+ */
128
+ export function cutText(text, max) {
129
+ if (text.length <= max)
130
+ return text;
131
+ const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
132
+ return text.slice(0, end);
133
+ }
134
+ function isHighSurrogate(c) {
135
+ return c >= 0xd800 && c <= 0xdbff;
136
+ }
137
+ /**
138
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
139
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
140
+ */
141
+ export function toWellFormed(text) {
142
+ return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
79
143
  }
80
144
  /** Base class for every error originating from this client. */
81
145
  export class CkanError extends Error {
@@ -134,6 +198,26 @@ export class CkanApiError extends CkanError {
134
198
  return this.status === 429 || this.status === 503;
135
199
  }
136
200
  }
201
+ /**
202
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
203
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
204
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
205
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
206
+ * or undefined when the error carries none.
207
+ */
208
+ export class CkanActionError extends CkanError {
209
+ action;
210
+ errorType;
211
+ constructor(message, args) {
212
+ super(message);
213
+ this.action = args.action;
214
+ this.errorType = args.errorType;
215
+ }
216
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
217
+ get isNotFound() {
218
+ return this.errorType === "Not Found Error";
219
+ }
220
+ }
137
221
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
138
222
  export class CkanNetworkError extends CkanError {
139
223
  }
@@ -1,11 +1,11 @@
1
1
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
9
  export { assertValid } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
@@ -1,9 +1,9 @@
1
1
  // Public entry point for the API client library.
2
2
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
7
  export { assertValid } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -2,7 +2,7 @@
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
3
  import { CkanClient, siteRoot } from "./client.js";
4
4
  import { validateBaseUrl } from "./engine.js";
5
- import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
5
+ import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
6
6
  import { PORTALS } from "./portals-list.js";
7
7
  import { assertValid, intRangeProblem } from "./validate.js";
8
8
  /**
@@ -118,9 +118,9 @@ function problemOf(err) {
118
118
  // The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
119
119
  const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
120
120
  if (reason !== undefined)
121
- return reason.slice(0, 80);
121
+ return cutText(reason, 80);
122
122
  }
123
- return message.slice(0, 80);
123
+ return cutText(message, 80);
124
124
  }
125
125
  /**
126
126
  * A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
@@ -2,7 +2,7 @@
2
2
  // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
3
  // enforce them with `assertValid` before any request; the CLI's value parsers call
4
4
  // the same functions, so a rule is written once and the CLI and the library agree.
5
- import { CkanValidationError } from "./errors.js";
5
+ import { CkanValidationError, cutForMessage } from "./errors.js";
6
6
  /**
7
7
  * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
8
  * something wrong with `value`; otherwise return `value` unchanged. A method that
@@ -147,7 +147,7 @@ export function baseUrlProblem(value) {
147
147
  return "Expected an absolute http(s) URL.";
148
148
  }
149
149
  if (url.protocol !== "http:" && url.protocol !== "https:") {
150
- return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
150
+ return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
151
151
  }
152
152
  if (/[?#]/.test(value))
153
153
  return "Expected a site URL without a query string or fragment.";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.2.0",
3
+ "version": "0.4.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {
@@ -18,7 +18,8 @@
18
18
  "dist/src",
19
19
  "!dist/src/**/*.map",
20
20
  "LICENSING.md",
21
- "CONTRIBUTING.md"
21
+ "CONTRIBUTING.md",
22
+ "DATA_LICENSE.md"
22
23
  ],
23
24
  "engines": {
24
25
  "node": ">=22.12"
@@ -29,7 +30,7 @@
29
30
  "prepack": "npm run build",
30
31
  "typecheck": "tsc -p tsconfig.json --noEmit",
31
32
  "pretest": "npm run build",
32
- "test": "node --test dist/test/*.test.js",
33
+ "test": "node --test --test-timeout=5000 dist/test/*.test.js",
33
34
  "docs": "node tools/docs/node_modules/typedoc/bin/typedoc src/index.ts --out out",
34
35
  "start": "node dist/src/cli/index.js",
35
36
  "update-portals": "npm run build && node dist/scripts/update-portals.js"