@maschinenlesbar.org/ckan-cli 0.0.5 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +28 -14
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +22 -9
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +18 -10
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +2 -1
- package/dist/src/cli/io.d.ts +17 -1
- package/dist/src/cli/io.js +22 -1
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +24 -6
- package/dist/src/cli/run.d.ts +0 -1
- package/dist/src/cli/run.js +9 -2
- package/dist/src/cli/shared.d.ts +28 -14
- package/dist/src/cli/shared.js +78 -40
- package/dist/src/client/client.d.ts +13 -6
- package/dist/src/client/client.js +100 -35
- package/dist/src/client/engine.d.ts +79 -6
- package/dist/src/client/engine.js +188 -74
- package/dist/src/client/errors.d.ts +13 -1
- package/dist/src/client/errors.js +30 -3
- package/dist/src/client/http.d.ts +0 -1
- package/dist/src/client/http.js +44 -36
- package/dist/src/client/index.d.ts +6 -5
- package/dist/src/client/index.js +4 -4
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +50 -3
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +5 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +55 -0
- package/dist/src/client/validate.js +119 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +2 -1
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
package/dist/src/cli/shared.js
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
1
|
// Shared helpers used across CLI command groups: option parsers, the global
|
|
2
2
|
// option resolver, and the JSON result renderer.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
|
+
import { isBidiControl } from "../client/engine.js";
|
|
5
|
+
import { CkanError } from "../client/errors.js";
|
|
4
6
|
import { findPortal } from "../client/portals.js";
|
|
7
|
+
import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
|
|
5
8
|
import { PORTALS } from "../client/portals-list.js";
|
|
6
9
|
/** commander value-parser: a non-negative integer. */
|
|
7
10
|
export function parseIntArg(value) {
|
|
@@ -12,58 +15,73 @@ export function parseIntArg(value) {
|
|
|
12
15
|
throw new InvalidArgumentError("Expected a non-negative integer.");
|
|
13
16
|
}
|
|
14
17
|
const n = Number(value);
|
|
15
|
-
//
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
18
|
+
// The library's count rule; it also rejects magnitudes that cannot be
|
|
19
|
+
// represented exactly (silent precision loss).
|
|
20
|
+
const reason = countProblem(n);
|
|
21
|
+
if (reason !== undefined)
|
|
22
|
+
throw new InvalidArgumentError(reason);
|
|
19
23
|
return n;
|
|
20
24
|
}
|
|
21
25
|
/**
|
|
22
|
-
* commander value-parser: a value that is not blank
|
|
23
|
-
*
|
|
26
|
+
* commander value-parser: a value that is not blank (the library's blankProblem
|
|
27
|
+
* rule, reported as a usage error before the client is built).
|
|
24
28
|
*/
|
|
25
29
|
export function parseNonEmpty(value) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
30
|
+
const reason = blankProblem(value);
|
|
31
|
+
if (reason !== undefined)
|
|
32
|
+
throw new InvalidArgumentError(reason);
|
|
33
|
+
return value;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
|
|
37
|
+
* the library's headerValueProblem rule (not blank, no control characters but tab,
|
|
38
|
+
* nothing above U+00FF), reported as a usage error.
|
|
39
|
+
*/
|
|
40
|
+
export function parseHeaderValue(value) {
|
|
41
|
+
const reason = headerValueProblem(value);
|
|
42
|
+
if (reason !== undefined)
|
|
43
|
+
throw new InvalidArgumentError(reason);
|
|
29
44
|
return value;
|
|
30
45
|
}
|
|
31
|
-
/**
|
|
46
|
+
/**
|
|
47
|
+
* commander value-parser for a text option's value: not blank, and not another
|
|
48
|
+
* long option. commander takes the next argument as the value of an option that
|
|
49
|
+
* needs one, even when it is a flag, so a forgotten value (`search --fq --rows 5`)
|
|
50
|
+
* silently shifted the rest into the query (`fq=--rows`, `q=5`) and still exited
|
|
51
|
+
* 0. No CKAN filter, sort, facet field or tag query starts with `--`; a single `-`
|
|
52
|
+
* (a negated Solr filter, `-organization:x`) stays allowed.
|
|
53
|
+
*/
|
|
54
|
+
export function parseOptionValue(value) {
|
|
55
|
+
if (value.startsWith("--")) {
|
|
56
|
+
throw new InvalidArgumentError("Expected a value, got another option. Give the option its value first.");
|
|
57
|
+
}
|
|
58
|
+
return parseNonEmpty(value);
|
|
59
|
+
}
|
|
60
|
+
/** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
|
|
32
61
|
export function collectNonEmpty(value, previous = []) {
|
|
33
|
-
return previous.concat([
|
|
62
|
+
return previous.concat([parseOptionValue(value)]);
|
|
34
63
|
}
|
|
35
|
-
/** Build a commander value-parser for a non-negative integer within [min, max]. */
|
|
64
|
+
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
36
65
|
export function parseBoundedInt(min, max) {
|
|
66
|
+
const problem = intRangeProblem(min, max);
|
|
37
67
|
return (value) => {
|
|
38
68
|
const n = parseIntArg(value);
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
throw new InvalidArgumentError(`Must be <= ${max}.`);
|
|
69
|
+
const reason = problem(n);
|
|
70
|
+
if (reason !== undefined)
|
|
71
|
+
throw new InvalidArgumentError(reason);
|
|
43
72
|
return n;
|
|
44
73
|
};
|
|
45
74
|
}
|
|
46
75
|
/**
|
|
47
|
-
* commander value-parser for --base-url
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
* but this surfaces the mistake up front and independently of the transport.
|
|
76
|
+
* commander value-parser for --base-url (and CKAN_BASE_URL, via the preAction hook
|
|
77
|
+
* in program.ts): the library's baseUrlProblem rule (not blank, no whitespace, an
|
|
78
|
+
* absolute http(s) URL, no query string or fragment), reported as a usage error
|
|
79
|
+
* before the client is built.
|
|
52
80
|
*/
|
|
53
81
|
export function parseBaseUrl(value) {
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
}
|
|
58
|
-
catch {
|
|
59
|
-
throw new InvalidArgumentError("Expected an absolute http(s) URL.");
|
|
60
|
-
}
|
|
61
|
-
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
62
|
-
throw new InvalidArgumentError(`Unsupported scheme "${url.protocol}". Expected an http(s) URL.`);
|
|
63
|
-
}
|
|
64
|
-
if (/[?#]/.test(value)) {
|
|
65
|
-
throw new InvalidArgumentError("Expected a site URL without a query string or fragment.");
|
|
66
|
-
}
|
|
82
|
+
const reason = baseUrlProblem(value);
|
|
83
|
+
if (reason !== undefined)
|
|
84
|
+
throw new InvalidArgumentError(reason);
|
|
67
85
|
return value;
|
|
68
86
|
}
|
|
69
87
|
/** commander value-parser for --portal: the URL of a portal in the built-in list. */
|
|
@@ -91,9 +109,10 @@ export function toEngineOptions(global) {
|
|
|
91
109
|
return options;
|
|
92
110
|
}
|
|
93
111
|
/**
|
|
94
|
-
* Escape the
|
|
95
|
-
* ESC) but not DEL
|
|
96
|
-
*
|
|
112
|
+
* Escape the characters JSON.stringify leaves raw although a terminal acts on them.
|
|
113
|
+
* It escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is
|
|
114
|
+
* the 8-bit form of CSI) or the bidi formatting characters (isBidiControl), which
|
|
115
|
+
* reorder the text that follows. The output is server data, so escape them; the
|
|
97
116
|
* result is equivalent, valid JSON (these characters only occur inside strings).
|
|
98
117
|
* Checked by char code so the source stays free of control bytes.
|
|
99
118
|
*/
|
|
@@ -102,16 +121,36 @@ export function escapeControlChars(json) {
|
|
|
102
121
|
let from = 0;
|
|
103
122
|
for (let i = 0; i < json.length; i++) {
|
|
104
123
|
const c = json.charCodeAt(i);
|
|
105
|
-
if (c >= 0x7f && c <= 0x9f) {
|
|
124
|
+
if ((c >= 0x7f && c <= 0x9f) || isBidiControl(c)) {
|
|
106
125
|
result += json.slice(from, i) + "\\u" + c.toString(16).padStart(4, "0");
|
|
107
126
|
from = i + 1;
|
|
108
127
|
}
|
|
109
128
|
}
|
|
110
129
|
return from === 0 ? json : result + json.slice(from);
|
|
111
130
|
}
|
|
131
|
+
/**
|
|
132
|
+
* JSON.stringify, pretty or compact. A deeply nested value (a hostile or broken
|
|
133
|
+
* response) overflows the stack — the pretty form far sooner than the compact one,
|
|
134
|
+
* which is why the message suggests --compact. The RangeError becomes a CkanError so
|
|
135
|
+
* the CLI prints a clear message instead of "Unexpected error: Maximum call stack
|
|
136
|
+
* size exceeded".
|
|
137
|
+
*/
|
|
138
|
+
function stringifyJson(value, compact) {
|
|
139
|
+
try {
|
|
140
|
+
return compact ? JSON.stringify(value) : JSON.stringify(value, null, 2);
|
|
141
|
+
}
|
|
142
|
+
catch (err) {
|
|
143
|
+
if (err instanceof RangeError) {
|
|
144
|
+
throw new CkanError(compact
|
|
145
|
+
? "The response is nested too deeply to print."
|
|
146
|
+
: "The response is nested too deeply to pretty-print; try --compact.", { cause: err });
|
|
147
|
+
}
|
|
148
|
+
throw err;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
112
151
|
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
113
152
|
export function renderJson(deps, global, value) {
|
|
114
|
-
const text = escapeControlChars(
|
|
153
|
+
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
115
154
|
deps.io.out(text);
|
|
116
155
|
}
|
|
117
156
|
/**
|
|
@@ -131,4 +170,3 @@ export function action(deps, fn) {
|
|
|
131
170
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
132
171
|
};
|
|
133
172
|
}
|
|
134
|
-
//# sourceMappingURL=shared.js.map
|
|
@@ -4,8 +4,9 @@ import type { Group, GroupListParams, JsonValue, License, ListParams, Organizati
|
|
|
4
4
|
/**
|
|
5
5
|
* The site root of a CKAN instance, from what a user is likely to paste: portals
|
|
6
6
|
* document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
|
|
7
|
-
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
8
|
-
*
|
|
7
|
+
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
8
|
+
* A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
|
|
9
|
+
* an http(s) URL, a query string or fragment) throws CkanValidationError.
|
|
9
10
|
*/
|
|
10
11
|
export declare function siteRoot(baseUrl: string): string;
|
|
11
12
|
export declare class CkanClient {
|
|
@@ -15,6 +16,12 @@ export declare class CkanClient {
|
|
|
15
16
|
constructor(options?: EngineOptions);
|
|
16
17
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
17
18
|
action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
|
|
19
|
+
/**
|
|
20
|
+
* An action whose `result` must have a known top-level shape (never a deep
|
|
21
|
+
* schema): a broken or foreign answer becomes a CkanParseError naming the
|
|
22
|
+
* action, not a TypeError further down (`page is not iterable`).
|
|
23
|
+
*/
|
|
24
|
+
private typed;
|
|
18
25
|
/** Site title, CKAN version and enabled extensions of the portal. */
|
|
19
26
|
status(): Promise<Status>;
|
|
20
27
|
/**
|
|
@@ -25,7 +32,8 @@ export declare class CkanClient {
|
|
|
25
32
|
* several as CKAN's `fq_list` (each its own Solr filter query; all must
|
|
26
33
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
27
34
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
28
|
-
* expects in `facet.field`.
|
|
35
|
+
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
36
|
+
* refused (CkanValidationError), never sent or dropped.
|
|
29
37
|
*/
|
|
30
38
|
packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
|
|
31
39
|
/** A single dataset by id or name. */
|
|
@@ -36,7 +44,7 @@ export declare class CkanClient {
|
|
|
36
44
|
groupShow(id: string): Promise<Group>;
|
|
37
45
|
/** A single resource (distribution) by id. */
|
|
38
46
|
resourceShow(id: string): Promise<Resource>;
|
|
39
|
-
/** Dataset names, paged with limit/offset. */
|
|
47
|
+
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
40
48
|
packageList(params?: ListParams): Promise<string[]>;
|
|
41
49
|
/**
|
|
42
50
|
* Organization names, or full objects with `all_fields`. See `groupOrOrgList`
|
|
@@ -45,7 +53,7 @@ export declare class CkanClient {
|
|
|
45
53
|
organizationList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
46
54
|
/** Group names, or full objects with `all_fields` (paged like organizationList). */
|
|
47
55
|
groupList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
48
|
-
/** Tags, optionally only those containing a substring. */
|
|
56
|
+
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
49
57
|
tagList(params?: TagListParams): Promise<string[]>;
|
|
50
58
|
/** The licences this portal offers. */
|
|
51
59
|
licenseList(): Promise<License[]>;
|
|
@@ -68,4 +76,3 @@ export declare class CkanClient {
|
|
|
68
76
|
*/
|
|
69
77
|
private show;
|
|
70
78
|
}
|
|
71
|
-
//# sourceMappingURL=client.d.ts.map
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
|
-
import { DEFAULT_BASE_URL, RequestEngine } from "./engine.js";
|
|
4
|
-
import { CkanError, CkanParseError, describeCkanError } from "./errors.js";
|
|
3
|
+
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
+
import { CkanError, CkanParseError, CkanValidationError, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
+
import { assertValid, blankProblem, countProblem, facetLimitProblem } from "./validate.js";
|
|
5
6
|
const ACTION = "/api/3/action";
|
|
6
7
|
/**
|
|
7
8
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
@@ -22,32 +23,66 @@ function entryKey(entry) {
|
|
|
22
23
|
return JSON.stringify(entry);
|
|
23
24
|
}
|
|
24
25
|
/**
|
|
25
|
-
* Drop undefined
|
|
26
|
-
*
|
|
27
|
-
*
|
|
26
|
+
* Drop undefined values so only the parameters the caller actually set are sent,
|
|
27
|
+
* and refuse a blank parameter name, a blank value or a list with a blank entry
|
|
28
|
+
* (CkanValidationError). CKAN reads an empty parameter as "not given", so a blank
|
|
29
|
+
* filter such as `tagList({ query: "" })` would silently return everything.
|
|
28
30
|
*/
|
|
29
31
|
function prune(params) {
|
|
30
|
-
|
|
32
|
+
// A null-prototype object, so a `__proto__` key is kept as a parameter instead
|
|
33
|
+
// of setting the prototype (and being lost).
|
|
34
|
+
const out = Object.create(null);
|
|
31
35
|
for (const [k, v] of Object.entries(params)) {
|
|
32
|
-
if (v === undefined
|
|
36
|
+
if (v === undefined)
|
|
33
37
|
continue;
|
|
34
|
-
|
|
38
|
+
assertValid("parameter name", k, blankProblem);
|
|
39
|
+
out[k] = assertValid(k, v, blankProblem);
|
|
35
40
|
}
|
|
36
41
|
return out;
|
|
37
42
|
}
|
|
43
|
+
/** A JSON object (not null, not an array). */
|
|
44
|
+
function isObject(value) {
|
|
45
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
46
|
+
}
|
|
47
|
+
/** A `package_search` result: an object with a numeric count and a results array. */
|
|
48
|
+
function isSearchResult(value) {
|
|
49
|
+
if (!isObject(value))
|
|
50
|
+
return false;
|
|
51
|
+
const count = value["count"];
|
|
52
|
+
return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 && Array.isArray(value["results"]);
|
|
53
|
+
}
|
|
54
|
+
/** The error for a `result` that does not have the shape the caller relies on. */
|
|
55
|
+
function shapeError(name, expected) {
|
|
56
|
+
return new CkanParseError(`Unexpected response shape from ${ACTION}/${name}: expected ${expected}.`);
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Check a `*_list` limit. CKAN reads `limit=0` as "no limit" (a whole catalogue,
|
|
60
|
+
* 245,893 names on Hamburg), while the `all_fields` pager would read it as "no
|
|
61
|
+
* entries"; so 0, like any non-positive or fractional value, is refused. To get
|
|
62
|
+
* the whole list, leave the limit out.
|
|
63
|
+
*/
|
|
64
|
+
function assertLimit(limit) {
|
|
65
|
+
if (limit === undefined)
|
|
66
|
+
return;
|
|
67
|
+
if (!Number.isSafeInteger(limit) || limit < 1) {
|
|
68
|
+
throw new CkanValidationError(`Invalid limit: expected a positive integer, got ${String(limit)}. Leave it out for the whole list.`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/** Check an optional count or offset (`rows`, `start`, `offset`): see countProblem. */
|
|
72
|
+
function assertCount(name, value) {
|
|
73
|
+
if (value !== undefined)
|
|
74
|
+
assertValid(name, value, countProblem);
|
|
75
|
+
}
|
|
38
76
|
/**
|
|
39
77
|
* The site root of a CKAN instance, from what a user is likely to paste: portals
|
|
40
78
|
* document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
|
|
41
|
-
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
42
|
-
*
|
|
79
|
+
* dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
|
|
80
|
+
* A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
|
|
81
|
+
* an http(s) URL, a query string or fragment) throws CkanValidationError.
|
|
43
82
|
*/
|
|
44
83
|
export function siteRoot(baseUrl) {
|
|
45
|
-
//
|
|
46
|
-
|
|
47
|
-
if (/[?#]/.test(baseUrl)) {
|
|
48
|
-
throw new CkanError(`The base URL must not contain a query string or fragment: ${baseUrl}`);
|
|
49
|
-
}
|
|
50
|
-
return baseUrl.replace(/\/+$/, "").replace(/\/api\/3(\/action)?$/, "");
|
|
84
|
+
// Checked before the suffix strip: a trailing space would hide the suffix from it.
|
|
85
|
+
return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
|
|
51
86
|
}
|
|
52
87
|
export class CkanClient {
|
|
53
88
|
engine;
|
|
@@ -66,16 +101,33 @@ export class CkanClient {
|
|
|
66
101
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
67
102
|
// envelope; say so rather than "failed: undefined".
|
|
68
103
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
69
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${this.site} a CKAN site?`);
|
|
104
|
+
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.site)} a CKAN site?`);
|
|
70
105
|
}
|
|
71
106
|
if (!env.success) {
|
|
72
|
-
|
|
107
|
+
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
108
|
+
// engine's error-detail sanitising: strip terminal controls here too.
|
|
109
|
+
throw new CkanError(`CKAN action "${name}" failed: ${sanitizeServerText(describeCkanError(env.error))}`);
|
|
73
110
|
}
|
|
111
|
+
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
112
|
+
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
113
|
+
if (env.result === undefined)
|
|
114
|
+
throw shapeError(name, "a result in the envelope");
|
|
74
115
|
return env.result;
|
|
75
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* An action whose `result` must have a known top-level shape (never a deep
|
|
119
|
+
* schema): a broken or foreign answer becomes a CkanParseError naming the
|
|
120
|
+
* action, not a TypeError further down (`page is not iterable`).
|
|
121
|
+
*/
|
|
122
|
+
async typed(name, params, ok, expected) {
|
|
123
|
+
const result = await this.action(name, params);
|
|
124
|
+
if (!ok(result))
|
|
125
|
+
throw shapeError(name, expected);
|
|
126
|
+
return result;
|
|
127
|
+
}
|
|
76
128
|
/** Site title, CKAN version and enabled extensions of the portal. */
|
|
77
129
|
status() {
|
|
78
|
-
return this.
|
|
130
|
+
return this.typed("status_show", {}, isObject, "a JSON object");
|
|
79
131
|
}
|
|
80
132
|
/**
|
|
81
133
|
* Full-text / faceted dataset search.
|
|
@@ -85,12 +137,21 @@ export class CkanClient {
|
|
|
85
137
|
* several as CKAN's `fq_list` (each its own Solr filter query; all must
|
|
86
138
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
87
139
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
88
|
-
* expects in `facet.field`.
|
|
140
|
+
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
141
|
+
* refused (CkanValidationError), never sent or dropped.
|
|
89
142
|
*/
|
|
90
|
-
packageSearch(params = {}) {
|
|
91
|
-
|
|
143
|
+
async packageSearch(params = {}) {
|
|
144
|
+
assertValid("q", params.q, blankProblem);
|
|
145
|
+
assertValid("fq", params.fq, blankProblem);
|
|
146
|
+
assertValid("sort", params.sort, blankProblem);
|
|
147
|
+
assertValid("facet_field", params.facet_field, blankProblem);
|
|
148
|
+
assertCount("rows", params.rows);
|
|
149
|
+
assertCount("start", params.start);
|
|
150
|
+
if (params.facet_limit !== undefined)
|
|
151
|
+
assertValid("facet_limit", params.facet_limit, facetLimitProblem);
|
|
152
|
+
const fq = params.fq ?? [];
|
|
92
153
|
const facetFields = params.facet_field ?? [];
|
|
93
|
-
return this.
|
|
154
|
+
return this.typed("package_search", {
|
|
94
155
|
q: params.q,
|
|
95
156
|
fq: fq.length === 1 ? fq[0] : undefined,
|
|
96
157
|
fq_list: fq.length > 1 ? fq : undefined,
|
|
@@ -99,7 +160,7 @@ export class CkanClient {
|
|
|
99
160
|
sort: params.sort,
|
|
100
161
|
"facet.field": facetFields.length > 0 ? JSON.stringify(facetFields) : undefined,
|
|
101
162
|
"facet.limit": params.facet_limit,
|
|
102
|
-
});
|
|
163
|
+
}, isSearchResult, "an object with a numeric count and a results array");
|
|
103
164
|
}
|
|
104
165
|
/** A single dataset by id or name. */
|
|
105
166
|
packageShow(id) {
|
|
@@ -117,9 +178,11 @@ export class CkanClient {
|
|
|
117
178
|
resourceShow(id) {
|
|
118
179
|
return this.show("resource_show", id);
|
|
119
180
|
}
|
|
120
|
-
/** Dataset names, paged with limit/offset. */
|
|
121
|
-
packageList(params = {}) {
|
|
122
|
-
|
|
181
|
+
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
182
|
+
async packageList(params = {}) {
|
|
183
|
+
assertLimit(params.limit);
|
|
184
|
+
assertCount("offset", params.offset);
|
|
185
|
+
return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
123
186
|
}
|
|
124
187
|
/**
|
|
125
188
|
* Organization names, or full objects with `all_fields`. See `groupOrOrgList`
|
|
@@ -132,13 +195,13 @@ export class CkanClient {
|
|
|
132
195
|
groupList(params = {}) {
|
|
133
196
|
return this.groupOrOrgList("group_list", params);
|
|
134
197
|
}
|
|
135
|
-
/** Tags, optionally only those containing a substring. */
|
|
198
|
+
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
136
199
|
tagList(params = {}) {
|
|
137
|
-
return this.
|
|
200
|
+
return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
|
|
138
201
|
}
|
|
139
202
|
/** The licences this portal offers. */
|
|
140
203
|
licenseList() {
|
|
141
|
-
return this.
|
|
204
|
+
return this.typed("license_list", {}, Array.isArray, "an array");
|
|
142
205
|
}
|
|
143
206
|
/**
|
|
144
207
|
* `organization_list` / `group_list`. CKAN caps an `all_fields` list at
|
|
@@ -153,19 +216,22 @@ export class CkanClient {
|
|
|
153
216
|
* that ignores `offset` cannot keep it going.
|
|
154
217
|
*/
|
|
155
218
|
async groupOrOrgList(action, params) {
|
|
219
|
+
assertLimit(params.limit);
|
|
220
|
+
// Before the first request, and before the all_fields pager steps on from it.
|
|
221
|
+
assertCount("offset", params.offset);
|
|
156
222
|
if (!params.all_fields) {
|
|
157
|
-
return this.
|
|
223
|
+
return this.typed(action, { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
158
224
|
}
|
|
159
225
|
const wanted = params.limit ?? Infinity;
|
|
160
226
|
let offset = params.offset ?? 0;
|
|
161
227
|
const seen = new Set();
|
|
162
228
|
const out = [];
|
|
163
229
|
while (out.length < wanted) {
|
|
164
|
-
const page = await this.
|
|
230
|
+
const page = await this.typed(action, {
|
|
165
231
|
all_fields: true,
|
|
166
232
|
limit: Math.min(ALL_FIELDS_PAGE, wanted - out.length),
|
|
167
233
|
offset: offset === 0 ? undefined : offset,
|
|
168
|
-
});
|
|
234
|
+
}, Array.isArray, "an array");
|
|
169
235
|
let added = 0;
|
|
170
236
|
for (const entry of page) {
|
|
171
237
|
const key = entryKey(entry);
|
|
@@ -188,7 +254,6 @@ export class CkanClient {
|
|
|
188
254
|
async show(action, id) {
|
|
189
255
|
if (id.trim() === "")
|
|
190
256
|
throw new CkanError(`${action} needs an id or name.`);
|
|
191
|
-
return this.
|
|
257
|
+
return this.typed(action, { id }, isObject, "a JSON object");
|
|
192
258
|
}
|
|
193
259
|
}
|
|
194
|
-
//# sourceMappingURL=client.js.map
|
|
@@ -13,24 +13,98 @@ export interface EngineOptions {
|
|
|
13
13
|
baseUrl?: string;
|
|
14
14
|
/** Swappable transport. Defaults to the built-in node http/https transport. */
|
|
15
15
|
transport?: Transport;
|
|
16
|
-
/**
|
|
16
|
+
/**
|
|
17
|
+
* Value of the User-Agent header. Must be sendable (see `assertHeaderValue`):
|
|
18
|
+
* not blank, no control characters but tab, nothing above U+00FF.
|
|
19
|
+
*/
|
|
17
20
|
userAgent?: string;
|
|
18
|
-
/** Per-request timeout in milliseconds
|
|
21
|
+
/** Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables. */
|
|
19
22
|
timeoutMs?: number;
|
|
20
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* Number of automatic retries for transient (429/503) responses. Each waits the
|
|
25
|
+
* response's `Retry-After` (up to `MAX_RETRY_AFTER_MS`; a longer one is not
|
|
26
|
+
* retried), or else `retryDelayMs * attempt`. 0 to `MAX_RETRIES`; defaults to 2.
|
|
27
|
+
*/
|
|
21
28
|
maxRetries?: number;
|
|
22
|
-
/** Base backoff between retries in milliseconds (grows linearly). */
|
|
29
|
+
/** Base backoff between retries in milliseconds (grows linearly); used without a Retry-After. */
|
|
23
30
|
retryDelayMs?: number;
|
|
24
|
-
/** Number of HTTP redirects (301/302/303/307/308) to follow
|
|
31
|
+
/** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
|
|
25
32
|
maxRedirects?: number;
|
|
26
33
|
/**
|
|
27
34
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
28
35
|
* from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
|
|
36
|
+
*
|
|
37
|
+
* Every numeric option must be a non-negative safe integer within its range;
|
|
38
|
+
* the constructor throws CkanValidationError otherwise (a negative or NaN
|
|
39
|
+
* timeout or cap would silently switch that guard off).
|
|
29
40
|
*/
|
|
30
41
|
maxResponseBytes?: number;
|
|
31
42
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
32
43
|
sleep?: (ms: number) => Promise<void>;
|
|
33
44
|
}
|
|
45
|
+
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
46
|
+
export declare const MAX_RETRIES = 10;
|
|
47
|
+
/** Most redirects the engine follows (`maxRedirects`). */
|
|
48
|
+
export declare const MAX_REDIRECTS = 10;
|
|
49
|
+
/**
|
|
50
|
+
* Check a value for an HTTP header (headerValueProblem) and return it, or throw
|
|
51
|
+
* CkanValidationError `Invalid <name>: …`.
|
|
52
|
+
*/
|
|
53
|
+
export declare function assertHeaderValue(name: string, value: string): string;
|
|
54
|
+
/**
|
|
55
|
+
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
56
|
+
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
57
|
+
* once: retrying early would only land inside the window the server asked us to wait
|
|
58
|
+
* out, and a hostile value must not stall the CLI.
|
|
59
|
+
*/
|
|
60
|
+
export declare const MAX_RETRY_AFTER_MS = 30000;
|
|
61
|
+
/**
|
|
62
|
+
* Parse a `Retry-After` header into a delay in milliseconds (RFC 9110 §10.2.3):
|
|
63
|
+
* either delay-seconds (`"120"`) or an HTTP-date (`"Wed, 21 Oct 2026 07:28:00 GMT"`,
|
|
64
|
+
* turned into the time left from `now`; a date in the past gives 0).
|
|
65
|
+
*
|
|
66
|
+
* Returns `undefined` when the header is absent or malformed — negative (`"-1"`),
|
|
67
|
+
* fractional (`"1.5"`), padded inside, any other date format — so the caller falls
|
|
68
|
+
* back to its own backoff. The strict patterns matter: `Date.parse` alone would
|
|
69
|
+
* read `"1.5"` as a date in 2001 and retry at once.
|
|
70
|
+
*/
|
|
71
|
+
export declare function parseRetryAfter(header: string | string[] | undefined, now?: number): number | undefined;
|
|
72
|
+
/**
|
|
73
|
+
* True for the Unicode bidirectional formatting characters: ALM (U+061C), LRM/RLM
|
|
74
|
+
* (U+200E/U+200F), the embeddings and overrides U+202A–U+202E and the isolates
|
|
75
|
+
* U+2066–U+2069. A terminal applies them to the text that follows, so an override
|
|
76
|
+
* in server text can reorder what the user sees ("Trojan Source" spoofing).
|
|
77
|
+
*/
|
|
78
|
+
export declare function isBidiControl(code: number): boolean;
|
|
79
|
+
/**
|
|
80
|
+
* Make a string that originates in an attacker-controlled response — the error
|
|
81
|
+
* `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
|
|
82
|
+
* into an error message on stderr:
|
|
83
|
+
*
|
|
84
|
+
* - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
|
|
85
|
+
* (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
|
|
86
|
+
* or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
|
|
87
|
+
* spoofing, title changes, OSC 52 clipboard writes).
|
|
88
|
+
* - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
|
|
89
|
+
* reorder the visible message.
|
|
90
|
+
* - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
|
|
91
|
+
* space and the ends are trimmed, so the text stays on one line and a server
|
|
92
|
+
* cannot forge an `Error:` line of its own.
|
|
93
|
+
*
|
|
94
|
+
* The CLI's JSON output is escaped separately (`escapeControlChars` in
|
|
95
|
+
* cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
|
|
96
|
+
* Written as a char-code filter so no raw control byte appears in this source.
|
|
97
|
+
*/
|
|
98
|
+
export declare function sanitizeServerText(text: string): string;
|
|
99
|
+
/**
|
|
100
|
+
* Check a base URL (baseUrlProblem: not blank, no whitespace or control
|
|
101
|
+
* characters, an absolute http(s) URL, no query string or fragment) and return it
|
|
102
|
+
* without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
|
|
103
|
+
* configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
|
|
104
|
+
* constructors call it, so a custom transport never sees a bad base URL; the
|
|
105
|
+
* default transport still re-checks the scheme on every hop.
|
|
106
|
+
*/
|
|
107
|
+
export declare function validateBaseUrl(raw: string): string;
|
|
34
108
|
export declare class RequestEngine {
|
|
35
109
|
private readonly baseUrl;
|
|
36
110
|
private readonly transport;
|
|
@@ -53,4 +127,3 @@ export declare class RequestEngine {
|
|
|
53
127
|
getJson<T>(path: string, query?: QueryParams): Promise<T>;
|
|
54
128
|
private toApiError;
|
|
55
129
|
}
|
|
56
|
-
//# sourceMappingURL=engine.d.ts.map
|