@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,9 @@
3
3
  /**
4
4
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
5
5
  * credential in a base URL never reaches an error message, a log or CI output.
6
- * A URL without userinfo, or one that does not parse, is returned unchanged.
6
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
7
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
8
+ * value without userinfo is returned unchanged.
7
9
  */
8
10
  export function redactUrl(url) {
9
11
  let parsed;
@@ -11,14 +13,134 @@ export function redactUrl(url) {
11
13
  parsed = new URL(url);
12
14
  }
13
15
  catch {
14
- return url;
16
+ // A value that doesn't parse can still carry credentials: cut them out by text.
17
+ return redactCredentials(url, credentialsIn(url));
15
18
  }
19
+ // A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
20
+ // the scheme "user:"), which is no URL with credentials at all.
16
21
  if (parsed.username === "" && parsed.password === "")
17
- return url;
22
+ return redactCredentials(url, credentialsIn(url));
18
23
  parsed.username = "***";
19
24
  parsed.password = "";
20
25
  return parsed.href;
21
26
  }
27
+ /**
28
+ * The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
29
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
30
+ * with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
31
+ * search text or a User-Agent as often as a credential, and the base URL always has a
32
+ * scheme. It works on URLs that don't parse too: the userinfo is everything between
33
+ * `://` and the last `@` before the host. Used to redact those exact strings from text
34
+ * that echoes the value (usage errors, help), whatever characters the password contains.
35
+ */
36
+ export function credentialsIn(value) {
37
+ const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
38
+ if (scheme === null)
39
+ return [];
40
+ const rest = value.slice(scheme[0].length);
41
+ let parses = false;
42
+ try {
43
+ new URL(value);
44
+ parses = true;
45
+ }
46
+ catch {
47
+ // Doesn't parse: the password may hold "/", "?", "#" or spaces.
48
+ }
49
+ // In a URL that parses, the userinfo ends at the last "@" of the authority (before the
50
+ // first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
51
+ const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
52
+ const end = authority.lastIndexOf("@");
53
+ return end > 0 ? [rest.slice(0, end)] : [];
54
+ }
55
+ /**
56
+ * The forms in which a server may echo the credentials of a userinfo (`user:password`,
57
+ * as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
58
+ * (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
59
+ * `user:password` itself, and the password alone when it is at least 4 characters long.
60
+ * `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
61
+ * are replaced as exact strings (`redactSecrets`).
62
+ */
63
+ export function echoedCredentialForms(userinfo) {
64
+ const colon = userinfo.indexOf(":");
65
+ if (colon < 0)
66
+ return [];
67
+ const decode = (part) => {
68
+ try {
69
+ return decodeURIComponent(part);
70
+ }
71
+ catch {
72
+ return part;
73
+ }
74
+ };
75
+ const user = decode(userinfo.slice(0, colon));
76
+ const password = decode(userinfo.slice(colon + 1));
77
+ if (password === "")
78
+ return [];
79
+ const pair = `${user}:${password}`;
80
+ const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
81
+ if (password.length >= 4)
82
+ forms.push(password);
83
+ return forms;
84
+ }
85
+ /** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
86
+ export function redactSecrets(text, secrets) {
87
+ let out = text;
88
+ for (const secret of secrets) {
89
+ if (secret === "")
90
+ continue;
91
+ out = out.split(secret).join("***");
92
+ }
93
+ return out;
94
+ }
95
+ /**
96
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
97
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
98
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
99
+ */
100
+ export function redactCredentials(text, credentials) {
101
+ let out = text;
102
+ for (const secret of credentials) {
103
+ if (secret === "")
104
+ continue;
105
+ out = out.split(`${secret}@`).join("***@");
106
+ }
107
+ return out;
108
+ }
109
+ /**
110
+ * Longest URL or server text (in characters) an error message shows. A long search
111
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
112
+ * log. The error's `url`, `detail` and `body` properties keep the full value. Every
113
+ * value an own message quotes from a server answer or the user's input (a redirect
114
+ * target, a charset, a Content-Type, an action name, a typed option value) is cut at
115
+ * this length too, so a library caller's `err.message` stays bounded.
116
+ */
117
+ export const MAX_MESSAGE_VALUE_LENGTH = 500;
118
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
119
+ export function cutForMessage(text) {
120
+ return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
121
+ }
122
+ /**
123
+ * `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
124
+ * would land after a high surrogate it is made one unit earlier, so a message that holds
125
+ * the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
126
+ * Text no longer than `max` is returned as it is; the caller marks a cut.
127
+ */
128
+ export function cutText(text, max) {
129
+ if (text.length <= max)
130
+ return text;
131
+ const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
132
+ return text.slice(0, end);
133
+ }
134
+ function isHighSurrogate(c) {
135
+ return c >= 0xd800 && c <= 0xdbff;
136
+ }
137
+ /**
138
+ * `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
139
+ * `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
140
+ */
141
+ export function toWellFormed(text) {
142
+ return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
143
+ }
22
144
  /** Base class for every error originating from this client. */
23
145
  export class CkanError extends Error {
24
146
  constructor(message, options) {
@@ -36,22 +158,66 @@ export class CkanApiError extends CkanError {
36
158
  url;
37
159
  method;
38
160
  body;
161
+ /** How many times the engine retried the request before giving up (0 when it did not). */
162
+ retries;
163
+ /**
164
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
165
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
166
+ * else undefined.
167
+ */
168
+ retryAfterMs;
39
169
  constructor(args) {
40
170
  // The URL is shown without userinfo: a credential in --base-url must not leak.
41
171
  const url = redactUrl(args.url);
42
- const detailPart = args.detail ? `: ${args.detail}` : "";
43
- super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
172
+ const parts = [];
173
+ if (args.detail)
174
+ parts.push(cutForMessage(args.detail));
175
+ if (args.retryAfterMs !== undefined) {
176
+ // Say why the retries the caller asked for never ran: the server asked for a
177
+ // wait longer than the engine sleeps, and retrying earlier would land inside it.
178
+ const wait = Math.ceil(args.retryAfterMs / 1000);
179
+ const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
180
+ parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
181
+ }
182
+ const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
183
+ const retries = args.retries ?? 0;
184
+ // Say that the status persisted through retries, so a user knows whether raising
185
+ // --max-retries could help.
186
+ const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
187
+ super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
44
188
  this.status = args.status;
45
189
  this.url = url;
46
190
  this.method = args.method;
47
191
  this.body = args.body;
48
192
  this.detail = args.detail;
193
+ this.retries = retries;
194
+ this.retryAfterMs = args.retryAfterMs;
49
195
  }
50
196
  /** True for statuses the API documents as transient and retry-able. */
51
197
  get isRetryable() {
52
198
  return this.status === 429 || this.status === 503;
53
199
  }
54
200
  }
201
+ /**
202
+ * CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
203
+ * answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
204
+ * message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
205
+ * CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
206
+ * or undefined when the error carries none.
207
+ */
208
+ export class CkanActionError extends CkanError {
209
+ action;
210
+ errorType;
211
+ constructor(message, args) {
212
+ super(message);
213
+ this.action = args.action;
214
+ this.errorType = args.errorType;
215
+ }
216
+ /** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
217
+ get isNotFound() {
218
+ return this.errorType === "Not Found Error";
219
+ }
220
+ }
55
221
  /** A transport-level failure (DNS, connection reset, timeout, ...). */
56
222
  export class CkanNetworkError extends CkanError {
57
223
  }
@@ -10,13 +10,36 @@ export interface HttpRequest {
10
10
  timeoutMs?: number;
11
11
  /** Hard cap on the response body size in bytes; the request aborts if exceeded. */
12
12
  maxResponseBytes?: number;
13
+ /**
14
+ * Aborted when the engine's overall deadline (`timeoutMs`) passes. A transport should
15
+ * stop the request then (`fetch(url, { signal })`); the engine rejects at the deadline
16
+ * either way, and enforces `maxResponseBytes` on the body it gets back, so neither
17
+ * limit depends on the transport.
18
+ */
19
+ signal?: AbortSignal;
20
+ /**
21
+ * Always `"manual"` from the engine: a transport must not follow redirects. The
22
+ * engine follows them itself and decides per hop whether the `Authorization`
23
+ * header (the base URL's userinfo) goes along (same origin only). A fetch-based
24
+ * transport passes it on: `fetch(url, { redirect })`.
25
+ */
26
+ redirect?: "manual";
13
27
  }
14
28
  export interface HttpResponse {
15
29
  status: number;
16
30
  headers: http.IncomingHttpHeaders;
17
31
  body: Buffer;
32
+ /**
33
+ * The URL the response came from, if the transport knows it (fetch's
34
+ * `response.url`). When it is on another origin than the request, the transport
35
+ * followed a redirect itself and the engine rejects the response with a
36
+ * CkanNetworkError.
37
+ */
38
+ url?: string;
18
39
  }
19
40
  export type Transport = (request: HttpRequest) => Promise<HttpResponse>;
41
+ /** The message for a body over the size cap, naming the option on both sides. */
42
+ export declare function sizeLimitMessage(maxBytes: number): string;
20
43
  /**
21
44
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
22
45
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -8,6 +8,10 @@
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
10
  import { CkanNetworkError, redactUrl } from "./errors.js";
11
+ /** The message for a body over the size cap, naming the option on both sides. */
12
+ export function sizeLimitMessage(maxBytes) {
13
+ return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
14
+ }
11
15
  /**
12
16
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
13
17
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -68,7 +72,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
68
72
  if (maxBytes !== undefined && received > maxBytes) {
69
73
  aborted = true;
70
74
  res.destroy();
71
- settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
75
+ settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
72
76
  return;
73
77
  }
74
78
  chunks.push(chunk);
@@ -109,6 +113,15 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
109
113
  // Don't let a pending deadline timer keep the event loop alive on its own.
110
114
  deadline.unref?.();
111
115
  }
116
+ if (request.signal !== undefined) {
117
+ const abort = () => {
118
+ req.destroy(new CkanNetworkError(`Request exceeded the ${request.timeoutMs ?? 0}ms deadline`));
119
+ };
120
+ if (request.signal.aborted)
121
+ abort();
122
+ else
123
+ request.signal.addEventListener("abort", abort, { once: true });
124
+ }
112
125
  req.on("error", (err) => {
113
126
  // A timeout destroy already passes an CkanNetworkError; don't double-wrap.
114
127
  settleReject(err instanceof CkanNetworkError ? err : new CkanNetworkError(err.message, { cause: err }));
@@ -1,11 +1,11 @@
1
- export { CkanClient, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
1
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
9
  export { assertValid } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
@@ -1,9 +1,9 @@
1
1
  // Public entry point for the API client library.
2
- export { CkanClient, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
2
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
7
  export { assertValid } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -1,5 +1,5 @@
1
1
  import { CkanClient } from "./client.js";
2
- import type { EngineOptions } from "./engine.js";
2
+ import { type EngineOptions } from "./engine.js";
3
3
  import type { Portal } from "./types.js";
4
4
  /**
5
5
  * One identity per portal, whatever form its URL was written in: host (without
@@ -1,7 +1,8 @@
1
1
  // Known CKAN portals: lookup and a short live check. The list itself is in
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
3
  import { CkanClient, siteRoot } from "./client.js";
4
- import { CkanApiError, CkanNetworkError, CkanParseError } from "./errors.js";
4
+ import { validateBaseUrl } from "./engine.js";
5
+ import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
5
6
  import { PORTALS } from "./portals-list.js";
6
7
  import { assertValid, intRangeProblem } from "./validate.js";
7
8
  /**
@@ -17,6 +18,10 @@ export function portalKey(url) {
17
18
  }
18
19
  /** A portal from `list` by its id (any case) or by any form of its URL. */
19
20
  export function findPortal(idOrUrl, list) {
21
+ if (typeof idOrUrl !== "string")
22
+ throw new CkanValidationError("Invalid portal: Expected a string.");
23
+ if (!Array.isArray(list))
24
+ throw new CkanValidationError("Invalid portal list: Expected an array.");
20
25
  const wanted = idOrUrl.trim().toLowerCase();
21
26
  const byId = list.find((p) => p.id.toLowerCase() === wanted);
22
27
  if (byId)
@@ -28,7 +33,14 @@ export function findPortal(idOrUrl, list) {
28
33
  catch {
29
34
  return undefined; // not a URL either
30
35
  }
31
- return list.find((p) => portalKey(p.url) === key);
36
+ return list.find((p) => {
37
+ try {
38
+ return portalKey(p.url) === key;
39
+ }
40
+ catch {
41
+ return false; // an entry without a usable url matches nothing
42
+ }
43
+ });
32
44
  }
33
45
  /**
34
46
  * A short live check of the CKAN portal a client points at. It works when `package_search?rows=0`
@@ -64,6 +76,17 @@ export async function checkPortal(client) {
64
76
  }
65
77
  return result;
66
78
  }
79
+ /**
80
+ * The first error `code` (`ENOTFOUND`, `ECONNREFUSED`, …) along a cause chain: the
81
+ * engine wraps a transport failure in a CkanNetworkError that names the request, so
82
+ * Node's code sits one or more levels down.
83
+ */
84
+ function errorCode(cause, depth = 0) {
85
+ if (typeof cause !== "object" || cause === null || depth > 5)
86
+ return undefined;
87
+ const code = cause.code;
88
+ return code !== undefined ? code : errorCode(cause.cause, depth + 1);
89
+ }
67
90
  /** A short, stable reason for a failed check. */
68
91
  function problemOf(err) {
69
92
  if (err instanceof CkanApiError) {
@@ -81,7 +104,7 @@ function problemOf(err) {
81
104
  return "invalid JSON";
82
105
  }
83
106
  if (err instanceof CkanNetworkError) {
84
- const code = err.cause?.code;
107
+ const code = errorCode(err.cause);
85
108
  if (code === "ENOTFOUND" || code === "EAI_AGAIN")
86
109
  return "host not found";
87
110
  if (code === "ECONNREFUSED")
@@ -92,8 +115,12 @@ function problemOf(err) {
92
115
  return "TLS certificate not verifiable";
93
116
  if (/timed out|deadline/.test(message))
94
117
  return "timeout";
118
+ // The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
119
+ const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
120
+ if (reason !== undefined)
121
+ return cutText(reason, 80);
95
122
  }
96
- return message.slice(0, 80);
123
+ return cutText(message, 80);
97
124
  }
98
125
  /**
99
126
  * A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
@@ -133,6 +160,8 @@ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
133
160
  * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
134
161
  */
135
162
  export async function checkPortalUrls(urls, options = {}) {
163
+ if (!Array.isArray(urls))
164
+ throw new CkanValidationError("Invalid urls: Expected an array.");
136
165
  const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
137
166
  const { retryDelayMs } = options;
138
167
  if (retryDelayMs !== undefined)
@@ -142,6 +171,10 @@ export async function checkPortalUrls(urls, options = {}) {
142
171
  return mapLimit(urls, concurrency, async (url) => {
143
172
  let client;
144
173
  try {
174
+ // Checked here, not left to the client: `baseUrl: undefined` selects the
175
+ // Hamburg default, so an entry without a url was checked against Hamburg and
176
+ // reported as working, with Hamburg's dataset count.
177
+ validateBaseUrl(url);
145
178
  client = createClient({ ...options.engineOptions, baseUrl: url });
146
179
  }
147
180
  catch (err) {
@@ -160,6 +193,8 @@ export async function checkPortalUrls(urls, options = {}) {
160
193
  * What `ckan portals --check` prints.
161
194
  */
162
195
  export async function checkPortals(portals = PORTALS, options = {}) {
196
+ if (!Array.isArray(portals))
197
+ throw new CkanValidationError("Invalid portals: Expected an array.");
163
198
  const date = options.date ?? new Date().toISOString().slice(0, 10);
164
199
  const checks = await checkPortalUrls(portals.map((p) => p.url), options);
165
200
  return portals.map((portal, i) => withCheck(portal, checks[i], date));
@@ -15,6 +15,16 @@ export declare function isBlank(value: string): boolean;
15
15
  * silently return the unfiltered result. Values that are not strings pass.
16
16
  */
17
17
  export declare function blankProblem(value: unknown): string | undefined;
18
+ /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
19
+ export declare function textProblem(value: unknown): string | undefined;
20
+ /** A list of non-blank strings (filter queries, facet fields). */
21
+ export declare function textListProblem(value: unknown): string | undefined;
22
+ /**
23
+ * A value the query-string builder can send: a string, a finite number, a boolean
24
+ * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
25
+ * function or `NaN` would go out as `[object Object]` or `NaN`.
26
+ */
27
+ export declare function queryValueProblem(value: unknown): string | undefined;
18
28
  /**
19
29
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
20
30
  * A negative, fractional or non-finite value would go upstream as given, and
@@ -47,7 +57,9 @@ export declare function headerValueProblem(value: string): string | undefined;
47
57
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
48
58
  * transport);
49
59
  * - no query string or fragment: the API path is appended, so it would land in
50
- * front of `/api/3/action`.
60
+ * front of `/api/3/action`;
61
+ * - a `%` in the user name or password must start a valid escape (`%25` for a
62
+ * literal one): the engine decodes the userinfo for the Authorization header.
51
63
  *
52
64
  * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
53
65
  * reasons never quote the value, so a credential in it cannot leak through them.
@@ -2,7 +2,7 @@
2
2
  // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
3
  // enforce them with `assertValid` before any request; the CLI's value parsers call
4
4
  // the same functions, so a rule is written once and the CLI and the library agree.
5
- import { CkanValidationError } from "./errors.js";
5
+ import { CkanValidationError, cutForMessage } from "./errors.js";
6
6
  /**
7
7
  * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
8
  * something wrong with `value`; otherwise return `value` unchanged. A method that
@@ -30,6 +30,36 @@ export function blankProblem(value) {
30
30
  return "Expected a non-empty value.";
31
31
  return undefined;
32
32
  }
33
+ /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
34
+ export function textProblem(value) {
35
+ if (typeof value !== "string")
36
+ return "Expected a string.";
37
+ return blankProblem(value);
38
+ }
39
+ /** A list of non-blank strings (filter queries, facet fields). */
40
+ export function textListProblem(value) {
41
+ if (!Array.isArray(value))
42
+ return "Expected a list of strings.";
43
+ if (value.some((v) => typeof v !== "string"))
44
+ return "Expected a list of strings.";
45
+ return blankProblem(value);
46
+ }
47
+ /**
48
+ * A value the query-string builder can send: a string, a finite number, a boolean
49
+ * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
50
+ * function or `NaN` would go out as `[object Object]` or `NaN`.
51
+ */
52
+ export function queryValueProblem(value) {
53
+ const ok = (v) => v === null ||
54
+ v === undefined ||
55
+ typeof v === "string" ||
56
+ typeof v === "boolean" ||
57
+ (typeof v === "number" && Number.isFinite(v)) ||
58
+ (v instanceof Date && !Number.isNaN(v.getTime()));
59
+ if (Array.isArray(value) ? value.every(ok) : ok(value))
60
+ return blankProblem(value);
61
+ return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
62
+ }
33
63
  /**
34
64
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
35
65
  * A negative, fractional or non-finite value would go upstream as given, and
@@ -65,6 +95,8 @@ export function intRangeProblem(min, max) {
65
95
  * so the source stays free of control bytes.
66
96
  */
67
97
  export function headerValueProblem(value) {
98
+ if (typeof value !== "string")
99
+ return "Expected a string.";
68
100
  const blank = blankProblem(value);
69
101
  if (blank !== undefined)
70
102
  return blank;
@@ -88,12 +120,16 @@ export function headerValueProblem(value) {
88
120
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
89
121
  * transport);
90
122
  * - no query string or fragment: the API path is appended, so it would land in
91
- * front of `/api/3/action`.
123
+ * front of `/api/3/action`;
124
+ * - a `%` in the user name or password must start a valid escape (`%25` for a
125
+ * literal one): the engine decodes the userinfo for the Authorization header.
92
126
  *
93
127
  * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
94
128
  * reasons never quote the value, so a credential in it cannot leak through them.
95
129
  */
96
130
  export function baseUrlProblem(value) {
131
+ if (typeof value !== "string")
132
+ return "Expected a string with an absolute http(s) URL.";
97
133
  if (isBlank(value))
98
134
  return "Expected an absolute http(s) URL.";
99
135
  if (value !== value.trim())
@@ -111,9 +147,19 @@ export function baseUrlProblem(value) {
111
147
  return "Expected an absolute http(s) URL.";
112
148
  }
113
149
  if (url.protocol !== "http:" && url.protocol !== "https:") {
114
- return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
150
+ return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
115
151
  }
116
152
  if (/[?#]/.test(value))
117
153
  return "Expected a site URL without a query string or fragment.";
154
+ // The engine decodes the userinfo into the Authorization header; a "%" that isn't an
155
+ // escape would fail there ("URI malformed") at request time. Reject it here.
156
+ for (const part of [url.username, url.password]) {
157
+ try {
158
+ decodeURIComponent(part);
159
+ }
160
+ catch {
161
+ return 'The user name or password has a "%" that is not followed by two hex digits; write a literal "%" as %25.';
162
+ }
163
+ }
118
164
  return undefined;
119
165
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.1.0",
3
+ "version": "0.4.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {
@@ -18,10 +18,11 @@
18
18
  "dist/src",
19
19
  "!dist/src/**/*.map",
20
20
  "LICENSING.md",
21
- "CONTRIBUTING.md"
21
+ "CONTRIBUTING.md",
22
+ "DATA_LICENSE.md"
22
23
  ],
23
24
  "engines": {
24
- "node": ">=20"
25
+ "node": ">=22.12"
25
26
  },
26
27
  "scripts": {
27
28
  "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
@@ -29,7 +30,7 @@
29
30
  "prepack": "npm run build",
30
31
  "typecheck": "tsc -p tsconfig.json --noEmit",
31
32
  "pretest": "npm run build",
32
- "test": "node --test dist/test/*.test.js",
33
+ "test": "node --test --test-timeout=5000 dist/test/*.test.js",
33
34
  "docs": "node tools/docs/node_modules/typedoc/bin/typedoc src/index.ts --out out",
34
35
  "start": "node dist/src/cli/index.js",
35
36
  "update-portals": "npm run build && node dist/scripts/update-portals.js"
@@ -66,7 +67,7 @@
66
67
  "commander": "15.0.0"
67
68
  },
68
69
  "devDependencies": {
69
- "@types/node": "^26.5.1",
70
+ "@types/node": "^22.20.5",
70
71
  "typescript": "7.0.2"
71
72
  }
72
73
  }