@maschinenlesbar.org/ckan-cli 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/io.d.ts +2 -1
- package/dist/src/cli/io.js +6 -2
- package/dist/src/cli/program.js +16 -9
- package/dist/src/cli/run.d.ts +27 -0
- package/dist/src/cli/run.js +82 -1
- package/dist/src/cli/shared.d.ts +8 -0
- package/dist/src/cli/shared.js +16 -0
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +123 -22
- package/dist/src/client/engine.d.ts +48 -9
- package/dist/src/client/engine.js +305 -45
- package/dist/src/client/errors.d.ts +37 -1
- package/dist/src/client/errors.js +87 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +2 -2
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +38 -3
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +47 -1
- package/package.json +3 -3
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
3
|
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
-
import { CkanError, CkanParseError, CkanValidationError, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
-
import { assertValid, blankProblem, countProblem, facetLimitProblem } from "./validate.js";
|
|
4
|
+
import { CkanError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
+
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
7
|
/**
|
|
8
8
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
@@ -13,6 +13,20 @@ const ACTION = "/api/3/action";
|
|
|
13
13
|
const ACTION_NAME = /^[a-z0-9_]+$/;
|
|
14
14
|
/** CKAN's default cap on an `all_fields` organization/group list. */
|
|
15
15
|
const ALL_FIELDS_PAGE = 25;
|
|
16
|
+
/**
|
|
17
|
+
* Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
|
|
18
|
+
* of 25). Real CKAN portals list a few hundred organizations or groups at most; a
|
|
19
|
+
* server whose pages keep returning new entries (a broken or hostile one) would
|
|
20
|
+
* otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
|
|
21
|
+
*/
|
|
22
|
+
export const MAX_ALL_FIELDS_PAGES = 400;
|
|
23
|
+
/**
|
|
24
|
+
* Pause between two `all_fields` pages, in milliseconds. A long list costs a few
|
|
25
|
+
* tenths of a second more; a server that never ends the list gets 10 requests a
|
|
26
|
+
* second at most instead of thousands.
|
|
27
|
+
*/
|
|
28
|
+
export const ALL_FIELDS_PAGE_DELAY_MS = 100;
|
|
29
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
16
30
|
/** A key for a list entry — its `id`, else its JSON — to drop duplicates across pages. */
|
|
17
31
|
function entryKey(entry) {
|
|
18
32
|
if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
|
|
@@ -29,6 +43,7 @@ function entryKey(entry) {
|
|
|
29
43
|
* filter such as `tagList({ query: "" })` would silently return everything.
|
|
30
44
|
*/
|
|
31
45
|
function prune(params) {
|
|
46
|
+
assertParams(params);
|
|
32
47
|
// A null-prototype object, so a `__proto__` key is kept as a parameter instead
|
|
33
48
|
// of setting the prototype (and being lost).
|
|
34
49
|
const out = Object.create(null);
|
|
@@ -36,10 +51,46 @@ function prune(params) {
|
|
|
36
51
|
if (v === undefined)
|
|
37
52
|
continue;
|
|
38
53
|
assertValid("parameter name", k, blankProblem);
|
|
39
|
-
|
|
54
|
+
// A value the query string can't carry (an object, NaN, a function) is refused,
|
|
55
|
+
// not sent as "[object Object]" or "NaN".
|
|
56
|
+
out[k] = assertValid(k, v, queryValueProblem);
|
|
40
57
|
}
|
|
41
58
|
return out;
|
|
42
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* A method's parameter object: a plain object (or undefined, for the defaults).
|
|
62
|
+
* `null`, a string or an array would otherwise fail as a raw TypeError.
|
|
63
|
+
*/
|
|
64
|
+
function assertParams(params) {
|
|
65
|
+
if (params === undefined)
|
|
66
|
+
return;
|
|
67
|
+
if (typeof params !== "object" || params === null || Array.isArray(params)) {
|
|
68
|
+
throw new CkanValidationError("Invalid params: Expected an object.");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Throw CkanValidationError unless every own key of `params` is in `known`. CKAN
|
|
73
|
+
* ignores a parameter it doesn't know, so a misspelled `fqs` or a JSON `__proto__`
|
|
74
|
+
* key would run the search unfiltered over the whole catalogue. Any other CKAN
|
|
75
|
+
* parameter goes through `action(name, params)`, which sends every key.
|
|
76
|
+
*/
|
|
77
|
+
function assertKeys(method, action, params, known) {
|
|
78
|
+
for (const key of Object.keys(params)) {
|
|
79
|
+
if (!known.includes(key)) {
|
|
80
|
+
throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
|
|
81
|
+
`Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_limit"];
|
|
86
|
+
const PACKAGE_LIST_KEYS = ["limit", "offset"];
|
|
87
|
+
const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
|
|
88
|
+
const TAG_LIST_KEYS = ["query"];
|
|
89
|
+
/** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
|
|
90
|
+
function assertText(name, value) {
|
|
91
|
+
if (value !== undefined)
|
|
92
|
+
assertValid(name, value, textProblem);
|
|
93
|
+
}
|
|
43
94
|
/** A JSON object (not null, not an array). */
|
|
44
95
|
function isObject(value) {
|
|
45
96
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -85,28 +136,35 @@ export function siteRoot(baseUrl) {
|
|
|
85
136
|
return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
|
|
86
137
|
}
|
|
87
138
|
export class CkanClient {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
139
|
+
// Real private fields (not TypeScript's `private`): console.log, util.inspect and
|
|
140
|
+
// JSON.stringify of a client never show them, so a password in the base URL can't
|
|
141
|
+
// be logged by accident.
|
|
142
|
+
#engine;
|
|
143
|
+
/** The site this client talks to, for error messages (always through redactUrl). */
|
|
144
|
+
#site;
|
|
145
|
+
/** The pause between `all_fields` pages (EngineOptions.sleep, for tests). */
|
|
146
|
+
#sleep;
|
|
91
147
|
constructor(options = {}) {
|
|
92
|
-
this
|
|
93
|
-
|
|
148
|
+
this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
149
|
+
// The engine checks every option, `sleep` included, before the client uses it.
|
|
150
|
+
this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
|
|
151
|
+
this.#sleep = options.sleep ?? realSleep;
|
|
94
152
|
}
|
|
95
153
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
96
154
|
async action(name, params = {}) {
|
|
97
|
-
if (!ACTION_NAME.test(name)) {
|
|
98
|
-
throw new
|
|
155
|
+
if (typeof name !== "string" || !ACTION_NAME.test(name)) {
|
|
156
|
+
throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
|
|
99
157
|
}
|
|
100
|
-
const env = await this
|
|
158
|
+
const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
|
|
101
159
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
102
160
|
// envelope; say so rather than "failed: undefined".
|
|
103
161
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
104
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this
|
|
162
|
+
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
105
163
|
}
|
|
106
164
|
if (!env.success) {
|
|
107
165
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
108
166
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
109
|
-
throw new CkanError(`CKAN action "${name}" failed: ${sanitizeServerText(describeCkanError(env.error))}`);
|
|
167
|
+
throw new CkanError(`CKAN action "${name}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`);
|
|
110
168
|
}
|
|
111
169
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
112
170
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -141,10 +199,14 @@ export class CkanClient {
|
|
|
141
199
|
* refused (CkanValidationError), never sent or dropped.
|
|
142
200
|
*/
|
|
143
201
|
async packageSearch(params = {}) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
202
|
+
assertParams(params);
|
|
203
|
+
assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
|
|
204
|
+
assertText("q", params.q);
|
|
205
|
+
if (params.fq !== undefined)
|
|
206
|
+
assertValid("fq", params.fq, textListProblem);
|
|
207
|
+
assertText("sort", params.sort);
|
|
208
|
+
if (params.facet_field !== undefined)
|
|
209
|
+
assertValid("facet_field", params.facet_field, textListProblem);
|
|
148
210
|
assertCount("rows", params.rows);
|
|
149
211
|
assertCount("start", params.start);
|
|
150
212
|
if (params.facet_limit !== undefined)
|
|
@@ -180,6 +242,8 @@ export class CkanClient {
|
|
|
180
242
|
}
|
|
181
243
|
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
182
244
|
async packageList(params = {}) {
|
|
245
|
+
assertParams(params);
|
|
246
|
+
assertKeys("packageList", "package_list", params, PACKAGE_LIST_KEYS);
|
|
183
247
|
assertLimit(params.limit);
|
|
184
248
|
assertCount("offset", params.offset);
|
|
185
249
|
return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
@@ -196,7 +260,10 @@ export class CkanClient {
|
|
|
196
260
|
return this.groupOrOrgList("group_list", params);
|
|
197
261
|
}
|
|
198
262
|
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
199
|
-
tagList(params = {}) {
|
|
263
|
+
async tagList(params = {}) {
|
|
264
|
+
assertParams(params);
|
|
265
|
+
assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
|
|
266
|
+
assertText("query", params.query);
|
|
200
267
|
return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
|
|
201
268
|
}
|
|
202
269
|
/** The licences this portal offers. */
|
|
@@ -214,8 +281,22 @@ export class CkanClient {
|
|
|
214
281
|
* some entries), so only an empty page ends the list. Entries are deduplicated
|
|
215
282
|
* (by `id`), and a page that adds nothing new ends the loop too, so a server
|
|
216
283
|
* that ignores `offset` cannot keep it going.
|
|
284
|
+
*
|
|
285
|
+
* Berlin counts hidden entries in its `limit`/`offset` window but leaves them
|
|
286
|
+
* out of the answer, so a window smaller than a full page (the last one before
|
|
287
|
+
* `limit` is reached) can come back empty although more entries follow. Such a
|
|
288
|
+
* miss is asked again at the same offset with a full page before it ends the list.
|
|
289
|
+
*
|
|
290
|
+
* Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
|
|
291
|
+
* `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
|
|
292
|
+
* keep returning new entries) the call fails with a CkanParseError.
|
|
217
293
|
*/
|
|
218
294
|
async groupOrOrgList(action, params) {
|
|
295
|
+
assertParams(params);
|
|
296
|
+
assertKeys(action === "group_list" ? "groupList" : "organizationList", action, params, GROUP_LIST_KEYS);
|
|
297
|
+
if (params.all_fields !== undefined && typeof params.all_fields !== "boolean") {
|
|
298
|
+
throw new CkanValidationError("Invalid all_fields: Expected a boolean.");
|
|
299
|
+
}
|
|
219
300
|
assertLimit(params.limit);
|
|
220
301
|
// Before the first request, and before the all_fields pager steps on from it.
|
|
221
302
|
assertCount("offset", params.offset);
|
|
@@ -226,12 +307,25 @@ export class CkanClient {
|
|
|
226
307
|
let offset = params.offset ?? 0;
|
|
227
308
|
const seen = new Set();
|
|
228
309
|
const out = [];
|
|
310
|
+
let pages = 0;
|
|
311
|
+
// Set after a short window came back empty (or with nothing new): from then on
|
|
312
|
+
// every page is a full one.
|
|
313
|
+
let fullPages = false;
|
|
229
314
|
while (out.length < wanted) {
|
|
315
|
+
if (pages >= MAX_ALL_FIELDS_PAGES) {
|
|
316
|
+
throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
|
|
317
|
+
`reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
|
|
318
|
+
`Pass a limit (--limit) to fetch part of it.`);
|
|
319
|
+
}
|
|
320
|
+
if (pages > 0)
|
|
321
|
+
await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
|
|
322
|
+
const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
|
|
230
323
|
const page = await this.typed(action, {
|
|
231
324
|
all_fields: true,
|
|
232
|
-
limit
|
|
325
|
+
limit,
|
|
233
326
|
offset: offset === 0 ? undefined : offset,
|
|
234
327
|
}, Array.isArray, "an array");
|
|
328
|
+
pages += 1;
|
|
235
329
|
let added = 0;
|
|
236
330
|
for (const entry of page) {
|
|
237
331
|
const key = entryKey(entry);
|
|
@@ -241,8 +335,15 @@ export class CkanClient {
|
|
|
241
335
|
out.push(entry);
|
|
242
336
|
added += 1;
|
|
243
337
|
}
|
|
244
|
-
if (page.length === 0 || added === 0)
|
|
338
|
+
if (page.length === 0 || added === 0) {
|
|
339
|
+
// A short window may have held only hidden entries: ask once more, same
|
|
340
|
+
// offset, for a full page before taking it as the end.
|
|
341
|
+
if (limit < ALL_FIELDS_PAGE) {
|
|
342
|
+
fullPages = true;
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
245
345
|
break;
|
|
346
|
+
}
|
|
246
347
|
offset += page.length;
|
|
247
348
|
}
|
|
248
349
|
return out.slice(0, wanted);
|
|
@@ -252,8 +353,8 @@ export class CkanClient {
|
|
|
252
353
|
* CKAN would answer with a 409 validation error instead.
|
|
253
354
|
*/
|
|
254
355
|
async show(action, id) {
|
|
255
|
-
if (id.trim() === "")
|
|
256
|
-
throw new
|
|
356
|
+
if (typeof id !== "string" || id.trim() === "")
|
|
357
|
+
throw new CkanValidationError(`${action} needs an id or name.`);
|
|
257
358
|
return this.typed(action, { id }, isObject, "a JSON object");
|
|
258
359
|
}
|
|
259
360
|
}
|
|
@@ -18,21 +18,32 @@ export interface EngineOptions {
|
|
|
18
18
|
* not blank, no control characters but tab, nothing above U+00FF.
|
|
19
19
|
*/
|
|
20
20
|
userAgent?: string;
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
|
|
23
|
+
* It covers the whole response, body included, and the engine enforces it for every
|
|
24
|
+
* transport (the request also gets an AbortSignal that fires at the deadline).
|
|
25
|
+
*/
|
|
22
26
|
timeoutMs?: number;
|
|
23
27
|
/**
|
|
24
|
-
* Number of automatic retries for transient (429/503) responses
|
|
25
|
-
*
|
|
26
|
-
*
|
|
28
|
+
* Number of automatic retries for transient (429/503) responses; a reset or other
|
|
29
|
+
* network error is not retried. Each waits `retryDelayMs * attempt`, or the
|
|
30
|
+
* response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
|
|
31
|
+
* one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
|
|
27
32
|
*/
|
|
28
33
|
maxRetries?: number;
|
|
29
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Base backoff between retries in milliseconds (grows linearly), 0 to
|
|
36
|
+
* `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
|
|
37
|
+
* `Retry-After`: the header can lengthen a wait, never shorten it.
|
|
38
|
+
*/
|
|
30
39
|
retryDelayMs?: number;
|
|
31
40
|
/** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
|
|
32
41
|
maxRedirects?: number;
|
|
33
42
|
/**
|
|
34
43
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
35
44
|
* from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
|
|
45
|
+
* Enforced by the engine for every transport: the built-in one aborts early, a
|
|
46
|
+
* custom one's body is checked when it arrives.
|
|
36
47
|
*
|
|
37
48
|
* Every numeric option must be a non-negative safe integer within its range;
|
|
38
49
|
* the constructor throws CkanValidationError otherwise (a negative or NaN
|
|
@@ -54,8 +65,8 @@ export declare function assertHeaderValue(name: string, value: string): string;
|
|
|
54
65
|
/**
|
|
55
66
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
56
67
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
57
|
-
* once: retrying early would only land inside the window
|
|
58
|
-
* out, and a hostile value must not stall the CLI.
|
|
68
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
69
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
59
70
|
*/
|
|
60
71
|
export declare const MAX_RETRY_AFTER_MS = 30000;
|
|
61
72
|
/**
|
|
@@ -106,7 +117,7 @@ export declare function sanitizeServerText(text: string): string;
|
|
|
106
117
|
*/
|
|
107
118
|
export declare function validateBaseUrl(raw: string): string;
|
|
108
119
|
export declare class RequestEngine {
|
|
109
|
-
private
|
|
120
|
+
#private;
|
|
110
121
|
private readonly transport;
|
|
111
122
|
private readonly userAgent;
|
|
112
123
|
private readonly timeoutMs;
|
|
@@ -116,8 +127,36 @@ export declare class RequestEngine {
|
|
|
116
127
|
private readonly maxResponseBytes;
|
|
117
128
|
private readonly sleep;
|
|
118
129
|
constructor(options?: EngineOptions);
|
|
119
|
-
/**
|
|
130
|
+
/**
|
|
131
|
+
* `text` without the base URL's credentials (raw and percent-decoded): server text
|
|
132
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
133
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
134
|
+
* messages built from server text.
|
|
135
|
+
*/
|
|
136
|
+
scrub(text: string): string;
|
|
137
|
+
/**
|
|
138
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
139
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
140
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
141
|
+
* base URL's password.
|
|
142
|
+
*/
|
|
143
|
+
private scrubCause;
|
|
144
|
+
/**
|
|
145
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
146
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
147
|
+
* Authorization header instead (see basicAuthorization).
|
|
148
|
+
*/
|
|
120
149
|
buildUrl(path: string, query?: QueryParams): string;
|
|
150
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
151
|
+
private composeUrl;
|
|
152
|
+
/**
|
|
153
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
154
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
155
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
156
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
157
|
+
* becomes a rejection.
|
|
158
|
+
*/
|
|
159
|
+
private callTransport;
|
|
121
160
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
122
161
|
request(method: string, path: string, options?: {
|
|
123
162
|
query?: QueryParams;
|