@maschinenlesbar.org/ckan-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,8 +1,8 @@
1
1
  // CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
2
2
  // Action API (`<site>/api/3/action`).
3
3
  import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
- import { CkanError, CkanParseError, CkanValidationError, describeCkanError, redactUrl } from "./errors.js";
5
- import { assertValid, blankProblem, countProblem, facetLimitProblem } from "./validate.js";
4
+ import { CkanError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
5
+ import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
6
6
  const ACTION = "/api/3/action";
7
7
  /**
8
8
  * CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
@@ -13,6 +13,20 @@ const ACTION = "/api/3/action";
13
13
  const ACTION_NAME = /^[a-z0-9_]+$/;
14
14
  /** CKAN's default cap on an `all_fields` organization/group list. */
15
15
  const ALL_FIELDS_PAGE = 25;
16
+ /**
17
+ * Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
18
+ * of 25). Real CKAN portals list a few hundred organizations or groups at most; a
19
+ * server whose pages keep returning new entries (a broken or hostile one) would
20
+ * otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
21
+ */
22
+ export const MAX_ALL_FIELDS_PAGES = 400;
23
+ /**
24
+ * Pause between two `all_fields` pages, in milliseconds. A long list costs a few
25
+ * tenths of a second more; a server that never ends the list gets 10 requests a
26
+ * second at most instead of thousands.
27
+ */
28
+ export const ALL_FIELDS_PAGE_DELAY_MS = 100;
29
+ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
16
30
  /** A key for a list entry — its `id`, else its JSON — to drop duplicates across pages. */
17
31
  function entryKey(entry) {
18
32
  if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
@@ -29,6 +43,7 @@ function entryKey(entry) {
29
43
  * filter such as `tagList({ query: "" })` would silently return everything.
30
44
  */
31
45
  function prune(params) {
46
+ assertParams(params);
32
47
  // A null-prototype object, so a `__proto__` key is kept as a parameter instead
33
48
  // of setting the prototype (and being lost).
34
49
  const out = Object.create(null);
@@ -36,10 +51,46 @@ function prune(params) {
36
51
  if (v === undefined)
37
52
  continue;
38
53
  assertValid("parameter name", k, blankProblem);
39
- out[k] = assertValid(k, v, blankProblem);
54
+ // A value the query string can't carry (an object, NaN, a function) is refused,
55
+ // not sent as "[object Object]" or "NaN".
56
+ out[k] = assertValid(k, v, queryValueProblem);
40
57
  }
41
58
  return out;
42
59
  }
60
+ /**
61
+ * A method's parameter object: a plain object (or undefined, for the defaults).
62
+ * `null`, a string or an array would otherwise fail as a raw TypeError.
63
+ */
64
+ function assertParams(params) {
65
+ if (params === undefined)
66
+ return;
67
+ if (typeof params !== "object" || params === null || Array.isArray(params)) {
68
+ throw new CkanValidationError("Invalid params: Expected an object.");
69
+ }
70
+ }
71
+ /**
72
+ * Throw CkanValidationError unless every own key of `params` is in `known`. CKAN
73
+ * ignores a parameter it doesn't know, so a misspelled `fqs` or a JSON `__proto__`
74
+ * key would run the search unfiltered over the whole catalogue. Any other CKAN
75
+ * parameter goes through `action(name, params)`, which sends every key.
76
+ */
77
+ function assertKeys(method, action, params, known) {
78
+ for (const key of Object.keys(params)) {
79
+ if (!known.includes(key)) {
80
+ throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
81
+ `Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
82
+ }
83
+ }
84
+ }
85
+ const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_limit"];
86
+ const PACKAGE_LIST_KEYS = ["limit", "offset"];
87
+ const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
88
+ const TAG_LIST_KEYS = ["query"];
89
+ /** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
90
+ function assertText(name, value) {
91
+ if (value !== undefined)
92
+ assertValid(name, value, textProblem);
93
+ }
43
94
  /** A JSON object (not null, not an array). */
44
95
  function isObject(value) {
45
96
  return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -85,28 +136,35 @@ export function siteRoot(baseUrl) {
85
136
  return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
86
137
  }
87
138
  export class CkanClient {
88
- engine;
89
- /** The site this client talks to, for error messages. */
90
- site;
139
+ // Real private fields (not TypeScript's `private`): console.log, util.inspect and
140
+ // JSON.stringify of a client never show them, so a password in the base URL can't
141
+ // be logged by accident.
142
+ #engine;
143
+ /** The site this client talks to, for error messages (always through redactUrl). */
144
+ #site;
145
+ /** The pause between `all_fields` pages (EngineOptions.sleep, for tests). */
146
+ #sleep;
91
147
  constructor(options = {}) {
92
- this.site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
93
- this.engine = new RequestEngine({ ...options, baseUrl: this.site });
148
+ this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
149
+ // The engine checks every option, `sleep` included, before the client uses it.
150
+ this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
151
+ this.#sleep = options.sleep ?? realSleep;
94
152
  }
95
153
  /** Call any CKAN action by name and return its unwrapped `result`. */
96
154
  async action(name, params = {}) {
97
- if (!ACTION_NAME.test(name)) {
98
- throw new CkanError(`Invalid CKAN action name: "${name}"`);
155
+ if (typeof name !== "string" || !ACTION_NAME.test(name)) {
156
+ throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
99
157
  }
100
- const env = await this.engine.getJson(`${ACTION}/${name}`, prune(params));
158
+ const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
101
159
  // Another JSON API at the same path (or a proxy's JSON error page) is not an
102
160
  // envelope; say so rather than "failed: undefined".
103
161
  if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
104
- throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.site)} a CKAN site?`);
162
+ throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
105
163
  }
106
164
  if (!env.success) {
107
165
  // A success:false envelope can come with HTTP 200, so it never passes the
108
166
  // engine's error-detail sanitising: strip terminal controls here too.
109
- throw new CkanError(`CKAN action "${name}" failed: ${sanitizeServerText(describeCkanError(env.error))}`);
167
+ throw new CkanError(`CKAN action "${name}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`);
110
168
  }
111
169
  // `{"success": true}` without a result would print nothing useful (and the CLI
112
170
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
@@ -141,10 +199,14 @@ export class CkanClient {
141
199
  * refused (CkanValidationError), never sent or dropped.
142
200
  */
143
201
  async packageSearch(params = {}) {
144
- assertValid("q", params.q, blankProblem);
145
- assertValid("fq", params.fq, blankProblem);
146
- assertValid("sort", params.sort, blankProblem);
147
- assertValid("facet_field", params.facet_field, blankProblem);
202
+ assertParams(params);
203
+ assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
204
+ assertText("q", params.q);
205
+ if (params.fq !== undefined)
206
+ assertValid("fq", params.fq, textListProblem);
207
+ assertText("sort", params.sort);
208
+ if (params.facet_field !== undefined)
209
+ assertValid("facet_field", params.facet_field, textListProblem);
148
210
  assertCount("rows", params.rows);
149
211
  assertCount("start", params.start);
150
212
  if (params.facet_limit !== undefined)
@@ -180,6 +242,8 @@ export class CkanClient {
180
242
  }
181
243
  /** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
182
244
  async packageList(params = {}) {
245
+ assertParams(params);
246
+ assertKeys("packageList", "package_list", params, PACKAGE_LIST_KEYS);
183
247
  assertLimit(params.limit);
184
248
  assertCount("offset", params.offset);
185
249
  return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
@@ -196,7 +260,10 @@ export class CkanClient {
196
260
  return this.groupOrOrgList("group_list", params);
197
261
  }
198
262
  /** Tags, optionally only those containing a substring (a blank one is refused). */
199
- tagList(params = {}) {
263
+ async tagList(params = {}) {
264
+ assertParams(params);
265
+ assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
266
+ assertText("query", params.query);
200
267
  return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
201
268
  }
202
269
  /** The licences this portal offers. */
@@ -214,8 +281,22 @@ export class CkanClient {
214
281
  * some entries), so only an empty page ends the list. Entries are deduplicated
215
282
  * (by `id`), and a page that adds nothing new ends the loop too, so a server
216
283
  * that ignores `offset` cannot keep it going.
284
+ *
285
+ * Berlin counts hidden entries in its `limit`/`offset` window but leaves them
286
+ * out of the answer, so a window smaller than a full page (the last one before
287
+ * `limit` is reached) can come back empty although more entries follow. Such a
288
+ * miss is asked again at the same offset with a full page before it ends the list.
289
+ *
290
+ * Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
291
+ * `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
292
+ * keep returning new entries) the call fails with a CkanParseError.
217
293
  */
218
294
  async groupOrOrgList(action, params) {
295
+ assertParams(params);
296
+ assertKeys(action === "group_list" ? "groupList" : "organizationList", action, params, GROUP_LIST_KEYS);
297
+ if (params.all_fields !== undefined && typeof params.all_fields !== "boolean") {
298
+ throw new CkanValidationError("Invalid all_fields: Expected a boolean.");
299
+ }
219
300
  assertLimit(params.limit);
220
301
  // Before the first request, and before the all_fields pager steps on from it.
221
302
  assertCount("offset", params.offset);
@@ -226,12 +307,25 @@ export class CkanClient {
226
307
  let offset = params.offset ?? 0;
227
308
  const seen = new Set();
228
309
  const out = [];
310
+ let pages = 0;
311
+ // Set after a short window came back empty (or with nothing new): from then on
312
+ // every page is a full one.
313
+ let fullPages = false;
229
314
  while (out.length < wanted) {
315
+ if (pages >= MAX_ALL_FIELDS_PAGES) {
316
+ throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
317
+ `reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
318
+ `Pass a limit (--limit) to fetch part of it.`);
319
+ }
320
+ if (pages > 0)
321
+ await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
322
+ const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
230
323
  const page = await this.typed(action, {
231
324
  all_fields: true,
232
- limit: Math.min(ALL_FIELDS_PAGE, wanted - out.length),
325
+ limit,
233
326
  offset: offset === 0 ? undefined : offset,
234
327
  }, Array.isArray, "an array");
328
+ pages += 1;
235
329
  let added = 0;
236
330
  for (const entry of page) {
237
331
  const key = entryKey(entry);
@@ -241,8 +335,15 @@ export class CkanClient {
241
335
  out.push(entry);
242
336
  added += 1;
243
337
  }
244
- if (page.length === 0 || added === 0)
338
+ if (page.length === 0 || added === 0) {
339
+ // A short window may have held only hidden entries: ask once more, same
340
+ // offset, for a full page before taking it as the end.
341
+ if (limit < ALL_FIELDS_PAGE) {
342
+ fullPages = true;
343
+ continue;
344
+ }
245
345
  break;
346
+ }
246
347
  offset += page.length;
247
348
  }
248
349
  return out.slice(0, wanted);
@@ -252,8 +353,8 @@ export class CkanClient {
252
353
  * CKAN would answer with a 409 validation error instead.
253
354
  */
254
355
  async show(action, id) {
255
- if (id.trim() === "")
256
- throw new CkanError(`${action} needs an id or name.`);
356
+ if (typeof id !== "string" || id.trim() === "")
357
+ throw new CkanValidationError(`${action} needs an id or name.`);
257
358
  return this.typed(action, { id }, isObject, "a JSON object");
258
359
  }
259
360
  }
@@ -18,21 +18,32 @@ export interface EngineOptions {
18
18
  * not blank, no control characters but tab, nothing above U+00FF.
19
19
  */
20
20
  userAgent?: string;
21
- /** Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables. */
21
+ /**
22
+ * Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
23
+ * It covers the whole response, body included, and the engine enforces it for every
24
+ * transport (the request also gets an AbortSignal that fires at the deadline).
25
+ */
22
26
  timeoutMs?: number;
23
27
  /**
24
- * Number of automatic retries for transient (429/503) responses. Each waits the
25
- * response's `Retry-After` (up to `MAX_RETRY_AFTER_MS`; a longer one is not
26
- * retried), or else `retryDelayMs * attempt`. 0 to `MAX_RETRIES`; defaults to 2.
28
+ * Number of automatic retries for transient (429/503) responses; a reset or other
29
+ * network error is not retried. Each waits `retryDelayMs * attempt`, or the
30
+ * response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
31
+ * one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
27
32
  */
28
33
  maxRetries?: number;
29
- /** Base backoff between retries in milliseconds (grows linearly); used without a Retry-After. */
34
+ /**
35
+ * Base backoff between retries in milliseconds (grows linearly), 0 to
36
+ * `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
37
+ * `Retry-After`: the header can lengthen a wait, never shorten it.
38
+ */
30
39
  retryDelayMs?: number;
31
40
  /** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
32
41
  maxRedirects?: number;
33
42
  /**
34
43
  * Hard cap on response body size in bytes (defends against memory exhaustion
35
44
  * from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
45
+ * Enforced by the engine for every transport: the built-in one aborts early, a
46
+ * custom one's body is checked when it arrives.
36
47
  *
37
48
  * Every numeric option must be a non-negative safe integer within its range;
38
49
  * the constructor throws CkanValidationError otherwise (a negative or NaN
@@ -54,8 +65,8 @@ export declare function assertHeaderValue(name: string, value: string): string;
54
65
  /**
55
66
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
56
67
  * server asks for longer, the engine does not retry at all and surfaces the error at
57
- * once: retrying early would only land inside the window the server asked us to wait
58
- * out, and a hostile value must not stall the CLI.
68
+ * once, naming the requested wait: retrying early would only land inside the window
69
+ * the server asked us to wait out, and a hostile value must not stall the CLI.
59
70
  */
60
71
  export declare const MAX_RETRY_AFTER_MS = 30000;
61
72
  /**
@@ -106,7 +117,7 @@ export declare function sanitizeServerText(text: string): string;
106
117
  */
107
118
  export declare function validateBaseUrl(raw: string): string;
108
119
  export declare class RequestEngine {
109
- private readonly baseUrl;
120
+ #private;
110
121
  private readonly transport;
111
122
  private readonly userAgent;
112
123
  private readonly timeoutMs;
@@ -116,8 +127,36 @@ export declare class RequestEngine {
116
127
  private readonly maxResponseBytes;
117
128
  private readonly sleep;
118
129
  constructor(options?: EngineOptions);
119
- /** Build a fully-qualified URL from a path and optional query parameters. */
130
+ /**
131
+ * `text` without the base URL's credentials (raw and percent-decoded): server text
132
+ * (an error body or a CKAN error that echoes the request URL) and transport text
133
+ * (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
134
+ * messages built from server text.
135
+ */
136
+ scrub(text: string): string;
137
+ /**
138
+ * A failure as the `cause` of the error the engine raises: the original when its
139
+ * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
140
+ * and the cause chain kept), so logging the error with its causes can't reveal the
141
+ * base URL's password.
142
+ */
143
+ private scrubCause;
144
+ /**
145
+ * Build a fully-qualified URL from a path and optional query parameters. It keeps
146
+ * the base URL's userinfo; `request()` leaves it out and sends it as an
147
+ * Authorization header instead (see basicAuthorization).
148
+ */
120
149
  buildUrl(path: string, query?: QueryParams): string;
150
+ /** buildUrl, with or without the base URL's userinfo. */
151
+ private composeUrl;
152
+ /**
153
+ * Call the transport under the overall deadline (`timeoutMs`): the request gets an
154
+ * AbortSignal that fires at the deadline, and the call rejects then whether the
155
+ * transport stops or not — a custom transport (fetch, a node:http wrapper) that
156
+ * ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
157
+ * becomes a rejection.
158
+ */
159
+ private callTransport;
121
160
  /** Perform a request with Accept negotiation and transient-error retries. */
122
161
  request(method: string, path: string, options?: {
123
162
  query?: QueryParams;