@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +23 -15
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +13 -12
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +0 -1
- package/dist/src/cli/io.d.ts +2 -2
- package/dist/src/cli/io.js +6 -3
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +17 -11
- package/dist/src/cli/run.d.ts +27 -1
- package/dist/src/cli/run.js +90 -2
- package/dist/src/cli/shared.d.ts +18 -15
- package/dist/src/cli/shared.js +45 -54
- package/dist/src/client/client.d.ts +30 -9
- package/dist/src/client/client.js +154 -37
- package/dist/src/client/engine.d.ts +74 -13
- package/dist/src/client/engine.js +341 -71
- package/dist/src/client/errors.d.ts +44 -2
- package/dist/src/client/errors.js +94 -6
- package/dist/src/client/http.d.ts +23 -1
- package/dist/src/client/http.js +54 -33
- package/dist/src/client/index.d.ts +7 -6
- package/dist/src/client/index.js +5 -5
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +86 -6
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +0 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +67 -0
- package/dist/src/client/validate.js +165 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +4 -3
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
package/dist/src/cli/shared.js
CHANGED
|
@@ -4,6 +4,7 @@ import { InvalidArgumentError } from "commander";
|
|
|
4
4
|
import { isBidiControl } from "../client/engine.js";
|
|
5
5
|
import { CkanError } from "../client/errors.js";
|
|
6
6
|
import { findPortal } from "../client/portals.js";
|
|
7
|
+
import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
|
|
7
8
|
import { PORTALS } from "../client/portals-list.js";
|
|
8
9
|
/** commander value-parser: a non-negative integer. */
|
|
9
10
|
export function parseIntArg(value) {
|
|
@@ -14,41 +15,32 @@ export function parseIntArg(value) {
|
|
|
14
15
|
throw new InvalidArgumentError("Expected a non-negative integer.");
|
|
15
16
|
}
|
|
16
17
|
const n = Number(value);
|
|
17
|
-
//
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
// The library's count rule; it also rejects magnitudes that cannot be
|
|
19
|
+
// represented exactly (silent precision loss).
|
|
20
|
+
const reason = countProblem(n);
|
|
21
|
+
if (reason !== undefined)
|
|
22
|
+
throw new InvalidArgumentError(reason);
|
|
21
23
|
return n;
|
|
22
24
|
}
|
|
23
25
|
/**
|
|
24
|
-
* commander value-parser: a value that is not blank
|
|
25
|
-
*
|
|
26
|
+
* commander value-parser: a value that is not blank (the library's blankProblem
|
|
27
|
+
* rule, reported as a usage error before the client is built).
|
|
26
28
|
*/
|
|
27
29
|
export function parseNonEmpty(value) {
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
30
|
+
const reason = blankProblem(value);
|
|
31
|
+
if (reason !== undefined)
|
|
32
|
+
throw new InvalidArgumentError(reason);
|
|
31
33
|
return value;
|
|
32
34
|
}
|
|
33
35
|
/**
|
|
34
|
-
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`)
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* U+00FF, which surfaced as "Unexpected error". Reject those here as a usage error,
|
|
38
|
-
* along with a blank value. Tab is allowed, as in HTTP. Checked by char code so the
|
|
39
|
-
* source stays free of control bytes.
|
|
36
|
+
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
|
|
37
|
+
* the library's headerValueProblem rule (not blank, no control characters but tab,
|
|
38
|
+
* nothing above U+00FF), reported as a usage error.
|
|
40
39
|
*/
|
|
41
40
|
export function parseHeaderValue(value) {
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
if ((c < 0x20 && c !== 0x09) || c === 0x7f) {
|
|
46
|
-
throw new InvalidArgumentError("Value contains control characters.");
|
|
47
|
-
}
|
|
48
|
-
if (c > 0xff) {
|
|
49
|
-
throw new InvalidArgumentError("Value contains characters outside Latin-1 (above U+00FF).");
|
|
50
|
-
}
|
|
51
|
-
}
|
|
41
|
+
const reason = headerValueProblem(value);
|
|
42
|
+
if (reason !== undefined)
|
|
43
|
+
throw new InvalidArgumentError(reason);
|
|
52
44
|
return value;
|
|
53
45
|
}
|
|
54
46
|
/**
|
|
@@ -65,47 +57,47 @@ export function parseOptionValue(value) {
|
|
|
65
57
|
}
|
|
66
58
|
return parseNonEmpty(value);
|
|
67
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* Wrap a commander value-parser for a single-valued option so that a second
|
|
62
|
+
* occurrence is a usage error. commander otherwise keeps only the last value, so
|
|
63
|
+
* `--rows 5 --rows 50` (or a second `--base-url`) silently dropped the first. The
|
|
64
|
+
* wrapper keeps its own count, so it also works for an option with a default (which
|
|
65
|
+
* commander hands the parser as `previous`); build it per program (`buildProgram`).
|
|
66
|
+
*/
|
|
67
|
+
export function once(parse) {
|
|
68
|
+
let seen = false;
|
|
69
|
+
return (value) => {
|
|
70
|
+
if (seen)
|
|
71
|
+
throw new InvalidArgumentError("Given more than once; this option takes a single value.");
|
|
72
|
+
seen = true;
|
|
73
|
+
return parse(value);
|
|
74
|
+
};
|
|
75
|
+
}
|
|
68
76
|
/** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
|
|
69
77
|
export function collectNonEmpty(value, previous = []) {
|
|
70
78
|
return previous.concat([parseOptionValue(value)]);
|
|
71
79
|
}
|
|
72
|
-
/** Build a commander value-parser for a non-negative integer within [min, max]. */
|
|
80
|
+
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
73
81
|
export function parseBoundedInt(min, max) {
|
|
82
|
+
const problem = intRangeProblem(min, max);
|
|
74
83
|
return (value) => {
|
|
75
84
|
const n = parseIntArg(value);
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
throw new InvalidArgumentError(`Must be <= ${max}.`);
|
|
85
|
+
const reason = problem(n);
|
|
86
|
+
if (reason !== undefined)
|
|
87
|
+
throw new InvalidArgumentError(reason);
|
|
80
88
|
return n;
|
|
81
89
|
};
|
|
82
90
|
}
|
|
83
91
|
/**
|
|
84
|
-
* commander value-parser for --base-url
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
88
|
-
* but this surfaces the mistake up front and independently of the transport.
|
|
92
|
+
* commander value-parser for --base-url (and CKAN_BASE_URL, via the preAction hook
|
|
93
|
+
* in program.ts): the library's baseUrlProblem rule (not blank, no whitespace, an
|
|
94
|
+
* absolute http(s) URL, no query string or fragment), reported as a usage error
|
|
95
|
+
* before the client is built.
|
|
89
96
|
*/
|
|
90
97
|
export function parseBaseUrl(value) {
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
}
|
|
95
|
-
catch {
|
|
96
|
-
throw new InvalidArgumentError("Expected an absolute http(s) URL.");
|
|
97
|
-
}
|
|
98
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
99
|
-
throw new InvalidArgumentError(`Unsupported scheme "${url.protocol}". Expected an http(s) URL.`);
|
|
100
|
-
}
|
|
101
|
-
if (/[?#]/.test(value)) {
|
|
102
|
-
throw new InvalidArgumentError("Expected a site URL without a query string or fragment.");
|
|
103
|
-
}
|
|
104
|
-
// `new URL` trims, but the value is used as given: a trailing space would land in
|
|
105
|
-
// the request path (`/404%20/api/3/...`) or the host.
|
|
106
|
-
if (value !== value.trim()) {
|
|
107
|
-
throw new InvalidArgumentError("A base URL cannot have surrounding whitespace.");
|
|
108
|
-
}
|
|
98
|
+
const reason = baseUrlProblem(value);
|
|
99
|
+
if (reason !== undefined)
|
|
100
|
+
throw new InvalidArgumentError(reason);
|
|
109
101
|
return value;
|
|
110
102
|
}
|
|
111
103
|
/** commander value-parser for --portal: the URL of a portal in the built-in list. */
|
|
@@ -194,4 +186,3 @@ export function action(deps, fn) {
|
|
|
194
186
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
195
187
|
};
|
|
196
188
|
}
|
|
197
|
-
//# sourceMappingURL=shared.js.map
|
|
@@ -1,17 +1,29 @@
|
|
|
1
1
|
import { type EngineOptions } from "./engine.js";
|
|
2
2
|
import type { QueryParams } from "./query.js";
|
|
3
3
|
import type { Group, GroupListParams, JsonValue, License, ListParams, Organization, Package, PackageSearchParams, PackageSearchResult, Resource, Status, TagListParams } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
|
|
6
|
+
* of 25). Real CKAN portals list a few hundred organizations or groups at most; a
|
|
7
|
+
* server whose pages keep returning new entries (a broken or hostile one) would
|
|
8
|
+
* otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
|
|
9
|
+
*/
|
|
10
|
+
export declare const MAX_ALL_FIELDS_PAGES = 400;
|
|
11
|
+
/**
|
|
12
|
+
* Pause between two `all_fields` pages, in milliseconds. A long list costs a few
|
|
13
|
+
* tenths of a second more; a server that never ends the list gets 10 requests a
|
|
14
|
+
* second at most instead of thousands.
|
|
15
|
+
*/
|
|
16
|
+
export declare const ALL_FIELDS_PAGE_DELAY_MS = 100;
|
|
4
17
|
/**
|
|
5
18
|
* The site root of a CKAN instance, from what a user is likely to paste: portals
|
|
6
19
|
* document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
|
|
7
|
-
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
8
|
-
*
|
|
20
|
+
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
21
|
+
* A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
|
|
22
|
+
* an http(s) URL, a query string or fragment) throws CkanValidationError.
|
|
9
23
|
*/
|
|
10
24
|
export declare function siteRoot(baseUrl: string): string;
|
|
11
25
|
export declare class CkanClient {
|
|
12
|
-
private
|
|
13
|
-
/** The site this client talks to, for error messages. */
|
|
14
|
-
private readonly site;
|
|
26
|
+
#private;
|
|
15
27
|
constructor(options?: EngineOptions);
|
|
16
28
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
17
29
|
action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
|
|
@@ -31,7 +43,8 @@ export declare class CkanClient {
|
|
|
31
43
|
* several as CKAN's `fq_list` (each its own Solr filter query; all must
|
|
32
44
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
33
45
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
34
|
-
* expects in `facet.field`.
|
|
46
|
+
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
47
|
+
* refused (CkanValidationError), never sent or dropped.
|
|
35
48
|
*/
|
|
36
49
|
packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
|
|
37
50
|
/** A single dataset by id or name. */
|
|
@@ -42,7 +55,7 @@ export declare class CkanClient {
|
|
|
42
55
|
groupShow(id: string): Promise<Group>;
|
|
43
56
|
/** A single resource (distribution) by id. */
|
|
44
57
|
resourceShow(id: string): Promise<Resource>;
|
|
45
|
-
/** Dataset names, paged with limit/offset (a positive limit
|
|
58
|
+
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
46
59
|
packageList(params?: ListParams): Promise<string[]>;
|
|
47
60
|
/**
|
|
48
61
|
* Organization names, or full objects with `all_fields`. See `groupOrOrgList`
|
|
@@ -51,7 +64,7 @@ export declare class CkanClient {
|
|
|
51
64
|
organizationList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
52
65
|
/** Group names, or full objects with `all_fields` (paged like organizationList). */
|
|
53
66
|
groupList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
54
|
-
/** Tags, optionally only those containing a substring. */
|
|
67
|
+
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
55
68
|
tagList(params?: TagListParams): Promise<string[]>;
|
|
56
69
|
/** The licences this portal offers. */
|
|
57
70
|
licenseList(): Promise<License[]>;
|
|
@@ -66,6 +79,15 @@ export declare class CkanClient {
|
|
|
66
79
|
* some entries), so only an empty page ends the list. Entries are deduplicated
|
|
67
80
|
* (by `id`), and a page that adds nothing new ends the loop too, so a server
|
|
68
81
|
* that ignores `offset` cannot keep it going.
|
|
82
|
+
*
|
|
83
|
+
* Berlin counts hidden entries in its `limit`/`offset` window but leaves them
|
|
84
|
+
* out of the answer, so a window smaller than a full page (the last one before
|
|
85
|
+
* `limit` is reached) can come back empty although more entries follow. Such a
|
|
86
|
+
* miss is asked again at the same offset with a full page before it ends the list.
|
|
87
|
+
*
|
|
88
|
+
* Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
|
|
89
|
+
* `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
|
|
90
|
+
* keep returning new entries) the call fails with a CkanParseError.
|
|
69
91
|
*/
|
|
70
92
|
private groupOrOrgList;
|
|
71
93
|
/**
|
|
@@ -74,4 +96,3 @@ export declare class CkanClient {
|
|
|
74
96
|
*/
|
|
75
97
|
private show;
|
|
76
98
|
}
|
|
77
|
-
//# sourceMappingURL=client.d.ts.map
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
|
-
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText } from "./engine.js";
|
|
4
|
-
import { CkanError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
|
|
3
|
+
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
+
import { CkanError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
+
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
5
6
|
const ACTION = "/api/3/action";
|
|
6
7
|
/**
|
|
7
8
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
@@ -12,6 +13,20 @@ const ACTION = "/api/3/action";
|
|
|
12
13
|
const ACTION_NAME = /^[a-z0-9_]+$/;
|
|
13
14
|
/** CKAN's default cap on an `all_fields` organization/group list. */
|
|
14
15
|
const ALL_FIELDS_PAGE = 25;
|
|
16
|
+
/**
|
|
17
|
+
* Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
|
|
18
|
+
* of 25). Real CKAN portals list a few hundred organizations or groups at most; a
|
|
19
|
+
* server whose pages keep returning new entries (a broken or hostile one) would
|
|
20
|
+
* otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
|
|
21
|
+
*/
|
|
22
|
+
export const MAX_ALL_FIELDS_PAGES = 400;
|
|
23
|
+
/**
|
|
24
|
+
* Pause between two `all_fields` pages, in milliseconds. A long list costs a few
|
|
25
|
+
* tenths of a second more; a server that never ends the list gets 10 requests a
|
|
26
|
+
* second at most instead of thousands.
|
|
27
|
+
*/
|
|
28
|
+
export const ALL_FIELDS_PAGE_DELAY_MS = 100;
|
|
29
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
15
30
|
/** A key for a list entry — its `id`, else its JSON — to drop duplicates across pages. */
|
|
16
31
|
function entryKey(entry) {
|
|
17
32
|
if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
|
|
@@ -22,21 +37,60 @@ function entryKey(entry) {
|
|
|
22
37
|
return JSON.stringify(entry);
|
|
23
38
|
}
|
|
24
39
|
/**
|
|
25
|
-
* Drop undefined
|
|
26
|
-
*
|
|
27
|
-
*
|
|
40
|
+
* Drop undefined values so only the parameters the caller actually set are sent,
|
|
41
|
+
* and refuse a blank parameter name, a blank value or a list with a blank entry
|
|
42
|
+
* (CkanValidationError). CKAN reads an empty parameter as "not given", so a blank
|
|
43
|
+
* filter such as `tagList({ query: "" })` would silently return everything.
|
|
28
44
|
*/
|
|
29
45
|
function prune(params) {
|
|
46
|
+
assertParams(params);
|
|
30
47
|
// A null-prototype object, so a `__proto__` key is kept as a parameter instead
|
|
31
48
|
// of setting the prototype (and being lost).
|
|
32
49
|
const out = Object.create(null);
|
|
33
50
|
for (const [k, v] of Object.entries(params)) {
|
|
34
|
-
if (v === undefined
|
|
51
|
+
if (v === undefined)
|
|
35
52
|
continue;
|
|
36
|
-
|
|
53
|
+
assertValid("parameter name", k, blankProblem);
|
|
54
|
+
// A value the query string can't carry (an object, NaN, a function) is refused,
|
|
55
|
+
// not sent as "[object Object]" or "NaN".
|
|
56
|
+
out[k] = assertValid(k, v, queryValueProblem);
|
|
37
57
|
}
|
|
38
58
|
return out;
|
|
39
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* A method's parameter object: a plain object (or undefined, for the defaults).
|
|
62
|
+
* `null`, a string or an array would otherwise fail as a raw TypeError.
|
|
63
|
+
*/
|
|
64
|
+
function assertParams(params) {
|
|
65
|
+
if (params === undefined)
|
|
66
|
+
return;
|
|
67
|
+
if (typeof params !== "object" || params === null || Array.isArray(params)) {
|
|
68
|
+
throw new CkanValidationError("Invalid params: Expected an object.");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Throw CkanValidationError unless every own key of `params` is in `known`. CKAN
|
|
73
|
+
* ignores a parameter it doesn't know, so a misspelled `fqs` or a JSON `__proto__`
|
|
74
|
+
* key would run the search unfiltered over the whole catalogue. Any other CKAN
|
|
75
|
+
* parameter goes through `action(name, params)`, which sends every key.
|
|
76
|
+
*/
|
|
77
|
+
function assertKeys(method, action, params, known) {
|
|
78
|
+
for (const key of Object.keys(params)) {
|
|
79
|
+
if (!known.includes(key)) {
|
|
80
|
+
throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
|
|
81
|
+
`Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_limit"];
|
|
86
|
+
const PACKAGE_LIST_KEYS = ["limit", "offset"];
|
|
87
|
+
const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
|
|
88
|
+
const TAG_LIST_KEYS = ["query"];
|
|
89
|
+
/** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
|
|
90
|
+
function assertText(name, value) {
|
|
91
|
+
if (value !== undefined)
|
|
92
|
+
assertValid(name, value, textProblem);
|
|
93
|
+
}
|
|
40
94
|
/** A JSON object (not null, not an array). */
|
|
41
95
|
function isObject(value) {
|
|
42
96
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -62,46 +116,55 @@ function assertLimit(limit) {
|
|
|
62
116
|
if (limit === undefined)
|
|
63
117
|
return;
|
|
64
118
|
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
65
|
-
throw new
|
|
119
|
+
throw new CkanValidationError(`Invalid limit: expected a positive integer, got ${String(limit)}. Leave it out for the whole list.`);
|
|
66
120
|
}
|
|
67
121
|
}
|
|
122
|
+
/** Check an optional count or offset (`rows`, `start`, `offset`): see countProblem. */
|
|
123
|
+
function assertCount(name, value) {
|
|
124
|
+
if (value !== undefined)
|
|
125
|
+
assertValid(name, value, countProblem);
|
|
126
|
+
}
|
|
68
127
|
/**
|
|
69
128
|
* The site root of a CKAN instance, from what a user is likely to paste: portals
|
|
70
129
|
* document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
|
|
71
|
-
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
72
|
-
*
|
|
130
|
+
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
131
|
+
* A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
|
|
132
|
+
* an http(s) URL, a query string or fragment) throws CkanValidationError.
|
|
73
133
|
*/
|
|
74
134
|
export function siteRoot(baseUrl) {
|
|
75
|
-
//
|
|
76
|
-
|
|
77
|
-
if (/[?#]/.test(baseUrl)) {
|
|
78
|
-
throw new CkanError(`The base URL must not contain a query string or fragment: ${redactUrl(baseUrl)}`);
|
|
79
|
-
}
|
|
80
|
-
return baseUrl.replace(/\/+$/, "").replace(/\/api\/3(\/action)?$/, "");
|
|
135
|
+
// Checked before the suffix strip: a trailing space would hide the suffix from it.
|
|
136
|
+
return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
|
|
81
137
|
}
|
|
82
138
|
export class CkanClient {
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
139
|
+
// Real private fields (not TypeScript's `private`): console.log, util.inspect and
|
|
140
|
+
// JSON.stringify of a client never show them, so a password in the base URL can't
|
|
141
|
+
// be logged by accident.
|
|
142
|
+
#engine;
|
|
143
|
+
/** The site this client talks to, for error messages (always through redactUrl). */
|
|
144
|
+
#site;
|
|
145
|
+
/** The pause between `all_fields` pages (EngineOptions.sleep, for tests). */
|
|
146
|
+
#sleep;
|
|
86
147
|
constructor(options = {}) {
|
|
87
|
-
this
|
|
88
|
-
|
|
148
|
+
this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
149
|
+
// The engine checks every option, `sleep` included, before the client uses it.
|
|
150
|
+
this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
|
|
151
|
+
this.#sleep = options.sleep ?? realSleep;
|
|
89
152
|
}
|
|
90
153
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
91
154
|
async action(name, params = {}) {
|
|
92
|
-
if (!ACTION_NAME.test(name)) {
|
|
93
|
-
throw new
|
|
155
|
+
if (typeof name !== "string" || !ACTION_NAME.test(name)) {
|
|
156
|
+
throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
|
|
94
157
|
}
|
|
95
|
-
const env = await this
|
|
158
|
+
const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
|
|
96
159
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
97
160
|
// envelope; say so rather than "failed: undefined".
|
|
98
161
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
99
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this
|
|
162
|
+
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
100
163
|
}
|
|
101
164
|
if (!env.success) {
|
|
102
165
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
103
166
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
104
|
-
throw new CkanError(`CKAN action "${name}" failed: ${sanitizeServerText(describeCkanError(env.error))}`);
|
|
167
|
+
throw new CkanError(`CKAN action "${name}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`);
|
|
105
168
|
}
|
|
106
169
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
107
170
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -132,10 +195,23 @@ export class CkanClient {
|
|
|
132
195
|
* several as CKAN's `fq_list` (each its own Solr filter query; all must
|
|
133
196
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
134
197
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
135
|
-
* expects in `facet.field`.
|
|
198
|
+
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
199
|
+
* refused (CkanValidationError), never sent or dropped.
|
|
136
200
|
*/
|
|
137
|
-
packageSearch(params = {}) {
|
|
138
|
-
|
|
201
|
+
async packageSearch(params = {}) {
|
|
202
|
+
assertParams(params);
|
|
203
|
+
assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
|
|
204
|
+
assertText("q", params.q);
|
|
205
|
+
if (params.fq !== undefined)
|
|
206
|
+
assertValid("fq", params.fq, textListProblem);
|
|
207
|
+
assertText("sort", params.sort);
|
|
208
|
+
if (params.facet_field !== undefined)
|
|
209
|
+
assertValid("facet_field", params.facet_field, textListProblem);
|
|
210
|
+
assertCount("rows", params.rows);
|
|
211
|
+
assertCount("start", params.start);
|
|
212
|
+
if (params.facet_limit !== undefined)
|
|
213
|
+
assertValid("facet_limit", params.facet_limit, facetLimitProblem);
|
|
214
|
+
const fq = params.fq ?? [];
|
|
139
215
|
const facetFields = params.facet_field ?? [];
|
|
140
216
|
return this.typed("package_search", {
|
|
141
217
|
q: params.q,
|
|
@@ -164,9 +240,12 @@ export class CkanClient {
|
|
|
164
240
|
resourceShow(id) {
|
|
165
241
|
return this.show("resource_show", id);
|
|
166
242
|
}
|
|
167
|
-
/** Dataset names, paged with limit/offset (a positive limit
|
|
243
|
+
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
168
244
|
async packageList(params = {}) {
|
|
245
|
+
assertParams(params);
|
|
246
|
+
assertKeys("packageList", "package_list", params, PACKAGE_LIST_KEYS);
|
|
169
247
|
assertLimit(params.limit);
|
|
248
|
+
assertCount("offset", params.offset);
|
|
170
249
|
return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
171
250
|
}
|
|
172
251
|
/**
|
|
@@ -180,8 +259,11 @@ export class CkanClient {
|
|
|
180
259
|
groupList(params = {}) {
|
|
181
260
|
return this.groupOrOrgList("group_list", params);
|
|
182
261
|
}
|
|
183
|
-
/** Tags, optionally only those containing a substring. */
|
|
184
|
-
tagList(params = {}) {
|
|
262
|
+
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
263
|
+
async tagList(params = {}) {
|
|
264
|
+
assertParams(params);
|
|
265
|
+
assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
|
|
266
|
+
assertText("query", params.query);
|
|
185
267
|
return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
|
|
186
268
|
}
|
|
187
269
|
/** The licences this portal offers. */
|
|
@@ -199,9 +281,25 @@ export class CkanClient {
|
|
|
199
281
|
* some entries), so only an empty page ends the list. Entries are deduplicated
|
|
200
282
|
* (by `id`), and a page that adds nothing new ends the loop too, so a server
|
|
201
283
|
* that ignores `offset` cannot keep it going.
|
|
284
|
+
*
|
|
285
|
+
* Berlin counts hidden entries in its `limit`/`offset` window but leaves them
|
|
286
|
+
* out of the answer, so a window smaller than a full page (the last one before
|
|
287
|
+
* `limit` is reached) can come back empty although more entries follow. Such a
|
|
288
|
+
* miss is asked again at the same offset with a full page before it ends the list.
|
|
289
|
+
*
|
|
290
|
+
* Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
|
|
291
|
+
* `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
|
|
292
|
+
* keep returning new entries) the call fails with a CkanParseError.
|
|
202
293
|
*/
|
|
203
294
|
async groupOrOrgList(action, params) {
|
|
295
|
+
assertParams(params);
|
|
296
|
+
assertKeys(action === "group_list" ? "groupList" : "organizationList", action, params, GROUP_LIST_KEYS);
|
|
297
|
+
if (params.all_fields !== undefined && typeof params.all_fields !== "boolean") {
|
|
298
|
+
throw new CkanValidationError("Invalid all_fields: Expected a boolean.");
|
|
299
|
+
}
|
|
204
300
|
assertLimit(params.limit);
|
|
301
|
+
// Before the first request, and before the all_fields pager steps on from it.
|
|
302
|
+
assertCount("offset", params.offset);
|
|
205
303
|
if (!params.all_fields) {
|
|
206
304
|
return this.typed(action, { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
207
305
|
}
|
|
@@ -209,12 +307,25 @@ export class CkanClient {
|
|
|
209
307
|
let offset = params.offset ?? 0;
|
|
210
308
|
const seen = new Set();
|
|
211
309
|
const out = [];
|
|
310
|
+
let pages = 0;
|
|
311
|
+
// Set after a short window came back empty (or with nothing new): from then on
|
|
312
|
+
// every page is a full one.
|
|
313
|
+
let fullPages = false;
|
|
212
314
|
while (out.length < wanted) {
|
|
315
|
+
if (pages >= MAX_ALL_FIELDS_PAGES) {
|
|
316
|
+
throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
|
|
317
|
+
`reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
|
|
318
|
+
`Pass a limit (--limit) to fetch part of it.`);
|
|
319
|
+
}
|
|
320
|
+
if (pages > 0)
|
|
321
|
+
await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
|
|
322
|
+
const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
|
|
213
323
|
const page = await this.typed(action, {
|
|
214
324
|
all_fields: true,
|
|
215
|
-
limit
|
|
325
|
+
limit,
|
|
216
326
|
offset: offset === 0 ? undefined : offset,
|
|
217
327
|
}, Array.isArray, "an array");
|
|
328
|
+
pages += 1;
|
|
218
329
|
let added = 0;
|
|
219
330
|
for (const entry of page) {
|
|
220
331
|
const key = entryKey(entry);
|
|
@@ -224,8 +335,15 @@ export class CkanClient {
|
|
|
224
335
|
out.push(entry);
|
|
225
336
|
added += 1;
|
|
226
337
|
}
|
|
227
|
-
if (page.length === 0 || added === 0)
|
|
338
|
+
if (page.length === 0 || added === 0) {
|
|
339
|
+
// A short window may have held only hidden entries: ask once more, same
|
|
340
|
+
// offset, for a full page before taking it as the end.
|
|
341
|
+
if (limit < ALL_FIELDS_PAGE) {
|
|
342
|
+
fullPages = true;
|
|
343
|
+
continue;
|
|
344
|
+
}
|
|
228
345
|
break;
|
|
346
|
+
}
|
|
229
347
|
offset += page.length;
|
|
230
348
|
}
|
|
231
349
|
return out.slice(0, wanted);
|
|
@@ -235,9 +353,8 @@ export class CkanClient {
|
|
|
235
353
|
* CKAN would answer with a 409 validation error instead.
|
|
236
354
|
*/
|
|
237
355
|
async show(action, id) {
|
|
238
|
-
if (id.trim() === "")
|
|
239
|
-
throw new
|
|
356
|
+
if (typeof id !== "string" || id.trim() === "")
|
|
357
|
+
throw new CkanValidationError(`${action} needs an id or name.`);
|
|
240
358
|
return this.typed(action, { id }, isObject, "a JSON object");
|
|
241
359
|
}
|
|
242
360
|
}
|
|
243
|
-
//# sourceMappingURL=client.js.map
|