@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +80 -17
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
|
@@ -7,7 +7,11 @@ export type Problem<T = unknown> = (value: T) => string | undefined;
|
|
|
7
7
|
* rather than throwing synchronously.
|
|
8
8
|
*/
|
|
9
9
|
export declare function assertValid<T>(name: string, value: T, problem: Problem<T>): T;
|
|
10
|
-
/**
|
|
10
|
+
/**
|
|
11
|
+
* True when nothing in `value` would match anything: it holds only whitespace, control and
|
|
12
|
+
* format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
|
|
13
|
+
* empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
|
|
14
|
+
*/
|
|
11
15
|
export declare function isBlank(value: string): boolean;
|
|
12
16
|
/**
|
|
13
17
|
* A blank string, or a list with a blank entry. CKAN treats an empty parameter as
|
|
@@ -15,25 +19,76 @@ export declare function isBlank(value: string): boolean;
|
|
|
15
19
|
* silently return the unfiltered result. Values that are not strings pass.
|
|
16
20
|
*/
|
|
17
21
|
export declare function blankProblem(value: unknown): string | undefined;
|
|
18
|
-
/**
|
|
19
|
-
|
|
22
|
+
/**
|
|
23
|
+
* Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
|
|
24
|
+
* an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
|
|
25
|
+
* would match nothing or everything:
|
|
26
|
+
* - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
|
|
27
|
+
* - a lone surrogate (half of a character, sent as U+FFFD);
|
|
28
|
+
* - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
|
|
29
|
+
* hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
|
|
30
|
+
* nothing, or something else (Solr splits a word at a zero-width space).
|
|
31
|
+
* The reason quotes the value with every invisible character escaped (`quoteValue`).
|
|
32
|
+
*/
|
|
33
|
+
export declare function textProblem(value: string): string | undefined;
|
|
20
34
|
/**
|
|
21
35
|
* A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
|
|
22
36
|
* with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
|
|
23
37
|
* "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
|
|
24
|
-
* composed form matches). Inner whitespace is kept. Used for
|
|
25
|
-
*
|
|
38
|
+
* composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
|
|
39
|
+
* query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
|
|
26
40
|
* NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
|
|
27
41
|
* meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
|
|
28
42
|
* forms) that a phrase or a wildcard may mean literally.
|
|
29
43
|
*/
|
|
30
44
|
export declare function normalizeText(value: string): string;
|
|
31
|
-
/**
|
|
32
|
-
|
|
45
|
+
/**
|
|
46
|
+
* Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
|
|
47
|
+
* field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
|
|
48
|
+
* with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
|
|
49
|
+
* only invisible characters or combining marks), no lone surrogate, and no control or format
|
|
50
|
+
* character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
|
|
51
|
+
* character in an id or a field name only ever makes it match nothing. The reason quotes the
|
|
52
|
+
* value visibly.
|
|
53
|
+
*/
|
|
54
|
+
export declare function tokenProblem(value: string): string | undefined;
|
|
55
|
+
/** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
|
|
56
|
+
export declare function normalizeToken(value: string): string;
|
|
57
|
+
/**
|
|
58
|
+
* Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
|
|
59
|
+
* (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
|
|
60
|
+
* no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
|
|
61
|
+
* hold a `/`; a value with one is a pasted URL or path, not an id.
|
|
62
|
+
*/
|
|
63
|
+
export declare function idProblem(value: string): string | undefined;
|
|
64
|
+
/**
|
|
65
|
+
* Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
|
|
66
|
+
* {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
|
|
67
|
+
* `extras_registerobject_type`). A field name with a space or another character names no field,
|
|
68
|
+
* and CKAN answers with an empty facet and exit 0.
|
|
69
|
+
*/
|
|
70
|
+
export declare function fieldProblem(value: string): string | undefined;
|
|
71
|
+
/**
|
|
72
|
+
* Why `value` cannot be sent as a sort expression, or undefined when it can: the
|
|
73
|
+
* {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
|
|
74
|
+
* and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
|
|
75
|
+
* refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
|
|
76
|
+
* not caught: CKAN sorts by its default then, as documented.
|
|
77
|
+
*/
|
|
78
|
+
export declare function sortProblem(value: string): string | undefined;
|
|
79
|
+
/**
|
|
80
|
+
* Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
|
|
81
|
+
* item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
|
|
82
|
+
* default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
|
|
83
|
+
*/
|
|
84
|
+
export declare function listProblem(items: readonly string[], itemProblem?: Problem<string>): string | undefined;
|
|
33
85
|
/**
|
|
34
86
|
* A value the query-string builder can send: a string, a finite number, a boolean
|
|
35
|
-
* or a Date, or a list of them (`null`/`undefined` mean "not given"). An
|
|
36
|
-
* function or `NaN` would go out as `[object Object]` or `NaN`.
|
|
87
|
+
* or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
|
|
88
|
+
* object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
|
|
89
|
+
* least one value and no `null`/`undefined` entry: the query builder drops those, so
|
|
90
|
+
* `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
|
|
91
|
+
* list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
|
|
37
92
|
*/
|
|
38
93
|
export declare function queryValueProblem(value: unknown): string | undefined;
|
|
39
94
|
/**
|
|
@@ -62,13 +117,17 @@ export declare function headerValueProblem(value: string): string | undefined;
|
|
|
62
117
|
*
|
|
63
118
|
* - not blank;
|
|
64
119
|
* - no surrounding or embedded whitespace and no control characters: `new URL`
|
|
65
|
-
* trims and drops tab/newline silently,
|
|
66
|
-
*
|
|
67
|
-
* (`/ckan%20/api/3/...`) or reach a custom transport raw;
|
|
120
|
+
* trims and drops tab/newline silently, so the URL requested would not be the one
|
|
121
|
+
* typed (a trailing space would request another path, `/ckan%20/api/3/...`);
|
|
68
122
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
69
|
-
* transport);
|
|
123
|
+
* transport); another scheme is named only when the value spells `<scheme>://`, since
|
|
124
|
+
* `user:password@host` parses with the user name as its scheme;
|
|
70
125
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
71
126
|
* front of `/api/3/action`;
|
|
127
|
+
* - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
|
|
128
|
+
* `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
|
|
129
|
+
* any slashes after the scheme, so text-based code would read another host or no userinfo
|
|
130
|
+
* (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
|
|
72
131
|
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
73
132
|
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
74
133
|
*
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the reason a value is invalid, or `undefined` when it is valid. Client methods
|
|
3
3
|
// enforce them with `assertValid` before any request; the CLI's value parsers call
|
|
4
4
|
// the same functions, so a rule is written once and the CLI and the library agree.
|
|
5
|
-
import { CkanValidationError, cutForMessage } from "./errors.js";
|
|
5
|
+
import { CkanValidationError, cutForMessage, quoteValue } from "./errors.js";
|
|
6
6
|
/**
|
|
7
7
|
* Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
|
|
8
8
|
* something wrong with `value`; otherwise return `value` unchanged. A method that
|
|
@@ -15,9 +15,17 @@ export function assertValid(name, value, problem) {
|
|
|
15
15
|
throw new CkanValidationError(`Invalid ${name}: ${reason}`);
|
|
16
16
|
return value;
|
|
17
17
|
}
|
|
18
|
-
/**
|
|
18
|
+
/** Characters that never make a value non-blank: whitespace, controls, format characters, combining marks. */
|
|
19
|
+
const INVISIBLE = /[\s\p{Cc}\p{Cf}\p{M}]/gu;
|
|
20
|
+
/** A lone surrogate: half of a character, which the query string would send as U+FFFD. */
|
|
21
|
+
const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
|
|
22
|
+
/**
|
|
23
|
+
* True when nothing in `value` would match anything: it holds only whitespace, control and
|
|
24
|
+
* format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
|
|
25
|
+
* empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
|
|
26
|
+
*/
|
|
19
27
|
export function isBlank(value) {
|
|
20
|
-
return value.
|
|
28
|
+
return value.replace(INVISIBLE, "") === "";
|
|
21
29
|
}
|
|
22
30
|
/**
|
|
23
31
|
* A blank string, or a list with a blank entry. CKAN treats an empty parameter as
|
|
@@ -30,18 +38,45 @@ export function blankProblem(value) {
|
|
|
30
38
|
return "Expected a non-empty value.";
|
|
31
39
|
return undefined;
|
|
32
40
|
}
|
|
33
|
-
/**
|
|
41
|
+
/** `U+200B` for the first character of `text` that `pattern` matches. */
|
|
42
|
+
function codePointOf(text, pattern) {
|
|
43
|
+
const ch = pattern.exec(text)?.[0] ?? "";
|
|
44
|
+
return `U+${(ch.codePointAt(0) ?? 0).toString(16).toUpperCase().padStart(4, "0")}`;
|
|
45
|
+
}
|
|
46
|
+
/** A control character inside a text value: C0 and C1, but not TAB, LF or CR (whitespace in a pasted phrase). */
|
|
47
|
+
const TEXT_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/;
|
|
48
|
+
/** A format character inside a text value, but not ZWNJ/ZWJ (U+200C/U+200D), which some scripts and emoji need. */
|
|
49
|
+
const TEXT_FORMAT = /(?![\u200c\u200d])\p{Cf}/u;
|
|
50
|
+
/**
|
|
51
|
+
* Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
|
|
52
|
+
* an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
|
|
53
|
+
* would match nothing or everything:
|
|
54
|
+
* - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
|
|
55
|
+
* - a lone surrogate (half of a character, sent as U+FFFD);
|
|
56
|
+
* - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
|
|
57
|
+
* hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
|
|
58
|
+
* nothing, or something else (Solr splits a word at a zero-width space).
|
|
59
|
+
* The reason quotes the value with every invisible character escaped (`quoteValue`).
|
|
60
|
+
*/
|
|
34
61
|
export function textProblem(value) {
|
|
35
62
|
if (typeof value !== "string")
|
|
36
63
|
return "Expected a string.";
|
|
37
|
-
|
|
64
|
+
if (isBlank(value))
|
|
65
|
+
return "Expected a non-empty value.";
|
|
66
|
+
if (LONE_SURROGATE.test(value))
|
|
67
|
+
return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
|
|
68
|
+
if (TEXT_CONTROL.test(value))
|
|
69
|
+
return `${quoteValue(value)} contains a control character (${codePointOf(value, TEXT_CONTROL)}).`;
|
|
70
|
+
if (TEXT_FORMAT.test(value))
|
|
71
|
+
return `${quoteValue(value)} contains an invisible character (${codePointOf(value, TEXT_FORMAT)}).`;
|
|
72
|
+
return undefined;
|
|
38
73
|
}
|
|
39
74
|
/**
|
|
40
75
|
* A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
|
|
41
76
|
* with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
|
|
42
77
|
* "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
|
|
43
|
-
* composed form matches). Inner whitespace is kept. Used for
|
|
44
|
-
*
|
|
78
|
+
* composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
|
|
79
|
+
* query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
|
|
45
80
|
* NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
|
|
46
81
|
* meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
|
|
47
82
|
* forms) that a phrase or a wildcard may mean literally.
|
|
@@ -49,29 +84,145 @@ export function textProblem(value) {
|
|
|
49
84
|
export function normalizeText(value) {
|
|
50
85
|
return value.trim().normalize("NFC");
|
|
51
86
|
}
|
|
52
|
-
/** A
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
87
|
+
/** A control or format character in a token (after trimming): every Cc and every Cf. */
|
|
88
|
+
const TOKEN_INVISIBLE = /[\p{Cc}\p{Cf}]/u;
|
|
89
|
+
/**
|
|
90
|
+
* Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
|
|
91
|
+
* field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
|
|
92
|
+
* with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
|
|
93
|
+
* only invisible characters or combining marks), no lone surrogate, and no control or format
|
|
94
|
+
* character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
|
|
95
|
+
* character in an id or a field name only ever makes it match nothing. The reason quotes the
|
|
96
|
+
* value visibly.
|
|
97
|
+
*/
|
|
98
|
+
export function tokenProblem(value) {
|
|
99
|
+
if (typeof value !== "string")
|
|
100
|
+
return "Expected a string.";
|
|
101
|
+
if (isBlank(value))
|
|
102
|
+
return "Expected a non-empty value.";
|
|
103
|
+
const token = value.trim();
|
|
104
|
+
if (LONE_SURROGATE.test(token))
|
|
105
|
+
return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
|
|
106
|
+
if (TOKEN_INVISIBLE.test(token)) {
|
|
107
|
+
return `${quoteValue(value)} contains an invisible or control character (${codePointOf(token, TOKEN_INVISIBLE)}).`;
|
|
108
|
+
}
|
|
109
|
+
return undefined;
|
|
110
|
+
}
|
|
111
|
+
/** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
|
|
112
|
+
export function normalizeToken(value) {
|
|
113
|
+
return value.trim().normalize("NFC");
|
|
114
|
+
}
|
|
115
|
+
/**
|
|
116
|
+
* Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
|
|
117
|
+
* (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
|
|
118
|
+
* no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
|
|
119
|
+
* hold a `/`; a value with one is a pasted URL or path, not an id.
|
|
120
|
+
*/
|
|
121
|
+
export function idProblem(value) {
|
|
122
|
+
const problem = tokenProblem(value);
|
|
123
|
+
if (problem !== undefined)
|
|
124
|
+
return problem;
|
|
125
|
+
const id = value.trim();
|
|
126
|
+
if (id.includes("/"))
|
|
127
|
+
return `${quoteValue(value)} is not an id or name: it contains "/".`;
|
|
128
|
+
if (id === "." || id === "..")
|
|
129
|
+
return '"." and ".." are not ids.';
|
|
130
|
+
return undefined;
|
|
131
|
+
}
|
|
132
|
+
/**
|
|
133
|
+
* Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
|
|
134
|
+
* {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
|
|
135
|
+
* `extras_registerobject_type`). A field name with a space or another character names no field,
|
|
136
|
+
* and CKAN answers with an empty facet and exit 0.
|
|
137
|
+
*/
|
|
138
|
+
export function fieldProblem(value) {
|
|
139
|
+
const problem = idProblem(value);
|
|
140
|
+
if (problem !== undefined)
|
|
141
|
+
return problem;
|
|
142
|
+
if (!/^[\w.-]+$/.test(value.trim()))
|
|
143
|
+
return `${quoteValue(value)} is not a Solr field name (letters, digits, _ . -), e.g. res_format.`;
|
|
144
|
+
return undefined;
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Why `value` cannot be sent as a sort expression, or undefined when it can: the
|
|
148
|
+
* {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
|
|
149
|
+
* and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
|
|
150
|
+
* refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
|
|
151
|
+
* not caught: CKAN sorts by its default then, as documented.
|
|
152
|
+
*/
|
|
153
|
+
export function sortProblem(value) {
|
|
154
|
+
const problem = tokenProblem(value);
|
|
155
|
+
if (problem !== undefined)
|
|
156
|
+
return problem;
|
|
157
|
+
// Split at the commas outside parentheses: a function sort (`div(a,b) desc`) holds commas.
|
|
158
|
+
const criteria = [];
|
|
159
|
+
let depth = 0;
|
|
160
|
+
let from = 0;
|
|
161
|
+
const sort = value.trim();
|
|
162
|
+
for (let i = 0; i < sort.length; i++) {
|
|
163
|
+
const c = sort[i];
|
|
164
|
+
if (c === "(")
|
|
165
|
+
depth++;
|
|
166
|
+
else if (c === ")")
|
|
167
|
+
depth--;
|
|
168
|
+
else if (c === "," && depth === 0) {
|
|
169
|
+
criteria.push(sort.slice(from, i));
|
|
170
|
+
from = i + 1;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
criteria.push(sort.slice(from));
|
|
174
|
+
if (criteria.some((criterion) => !/^\s*\S.*\s(asc|desc)\s*$/i.test(criterion))) {
|
|
175
|
+
return `${quoteValue(value)} is not a sort expression: expected "<field> asc|desc[, <field> asc|desc]", e.g. "metadata_modified desc".`;
|
|
176
|
+
}
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
|
|
181
|
+
* item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
|
|
182
|
+
* default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
|
|
183
|
+
*/
|
|
184
|
+
export function listProblem(items, itemProblem = tokenProblem) {
|
|
185
|
+
if (!Array.isArray(items))
|
|
57
186
|
return "Expected a list of strings.";
|
|
58
|
-
|
|
187
|
+
if (items.length === 0)
|
|
188
|
+
return "Expected at least one value.";
|
|
189
|
+
for (let i = 0; i < items.length; i++) {
|
|
190
|
+
const problem = itemProblem(items[i]);
|
|
191
|
+
if (problem !== undefined)
|
|
192
|
+
return items.length === 1 ? problem : `item ${i + 1} of ${items.length}: ${problem}`;
|
|
193
|
+
}
|
|
194
|
+
return undefined;
|
|
59
195
|
}
|
|
60
196
|
/**
|
|
61
197
|
* A value the query-string builder can send: a string, a finite number, a boolean
|
|
62
|
-
* or a Date, or a list of them (`null`/`undefined` mean "not given"). An
|
|
63
|
-
* function or `NaN` would go out as `[object Object]` or `NaN`.
|
|
198
|
+
* or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
|
|
199
|
+
* object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
|
|
200
|
+
* least one value and no `null`/`undefined` entry: the query builder drops those, so
|
|
201
|
+
* `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
|
|
202
|
+
* list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
|
|
64
203
|
*/
|
|
65
204
|
export function queryValueProblem(value) {
|
|
205
|
+
if (Array.isArray(value)) {
|
|
206
|
+
if (value.length === 0)
|
|
207
|
+
return "Expected at least one value; leave the parameter out instead.";
|
|
208
|
+
const missing = value.findIndex((v) => v === null || v === undefined);
|
|
209
|
+
if (missing >= 0) {
|
|
210
|
+
return `item ${missing + 1} of ${value.length} is ${String(value[missing])}; a list holds values only (leave the parameter out instead).`;
|
|
211
|
+
}
|
|
212
|
+
}
|
|
66
213
|
const ok = (v) => v === null ||
|
|
67
214
|
v === undefined ||
|
|
68
215
|
typeof v === "string" ||
|
|
69
216
|
typeof v === "boolean" ||
|
|
70
217
|
(typeof v === "number" && Number.isFinite(v)) ||
|
|
71
218
|
(v instanceof Date && !Number.isNaN(v.getTime()));
|
|
72
|
-
if (Array.isArray(value) ? value.every(ok) : ok(value))
|
|
73
|
-
return
|
|
74
|
-
|
|
219
|
+
if (!(Array.isArray(value) ? value.every(ok) : ok(value))) {
|
|
220
|
+
return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
|
|
221
|
+
}
|
|
222
|
+
// A string goes out as text (normalizeText): the text rules apply to it and to each list entry.
|
|
223
|
+
const strings = (Array.isArray(value) ? value : [value]).filter((v) => typeof v === "string");
|
|
224
|
+
const problem = listProblem(strings, textProblem);
|
|
225
|
+
return strings.length === 0 ? undefined : problem;
|
|
75
226
|
}
|
|
76
227
|
/**
|
|
77
228
|
* A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
|
|
@@ -127,13 +278,17 @@ export function headerValueProblem(value) {
|
|
|
127
278
|
*
|
|
128
279
|
* - not blank;
|
|
129
280
|
* - no surrounding or embedded whitespace and no control characters: `new URL`
|
|
130
|
-
* trims and drops tab/newline silently,
|
|
131
|
-
*
|
|
132
|
-
* (`/ckan%20/api/3/...`) or reach a custom transport raw;
|
|
281
|
+
* trims and drops tab/newline silently, so the URL requested would not be the one
|
|
282
|
+
* typed (a trailing space would request another path, `/ckan%20/api/3/...`);
|
|
133
283
|
* - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
|
|
134
|
-
* transport);
|
|
284
|
+
* transport); another scheme is named only when the value spells `<scheme>://`, since
|
|
285
|
+
* `user:password@host` parses with the user name as its scheme;
|
|
135
286
|
* - no query string or fragment: the API path is appended, so it would land in
|
|
136
287
|
* front of `/api/3/action`;
|
|
288
|
+
* - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
|
|
289
|
+
* `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
|
|
290
|
+
* any slashes after the scheme, so text-based code would read another host or no userinfo
|
|
291
|
+
* (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
|
|
137
292
|
* - a `%` in the user name or password must start a valid escape (`%25` for a
|
|
138
293
|
* literal one): the engine decodes the userinfo for the Authorization header.
|
|
139
294
|
*
|
|
@@ -160,10 +315,23 @@ export function baseUrlProblem(value) {
|
|
|
160
315
|
return "Expected an absolute http(s) URL.";
|
|
161
316
|
}
|
|
162
317
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
318
|
+
// `user:password@host` without a scheme parses as a URL whose scheme is the user name
|
|
319
|
+
// ("t0kens3cret:"), lower-cased, which no exact-string redaction can match: name a scheme
|
|
320
|
+
// only when the value spells one, `<scheme>://`.
|
|
321
|
+
if (!/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
|
|
322
|
+
return "Expected an absolute http(s) URL.";
|
|
163
323
|
return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
|
|
164
324
|
}
|
|
165
325
|
if (/[?#]/.test(value))
|
|
166
326
|
return "Expected a site URL without a query string or fragment.";
|
|
327
|
+
// WHATWG reads a "\" in an http(s) URL as "/", so it ends the host, and it skips any number of
|
|
328
|
+
// slashes after "http:": `http://u:pw@a.example\@b.example` is a.example to the parser and
|
|
329
|
+
// was b.example to text-based code, and `http:u:pw@host` hid the userinfo from the redaction.
|
|
330
|
+
// Every reader of the base URL has to agree where its host is.
|
|
331
|
+
if (value.includes("\\"))
|
|
332
|
+
return 'A base URL cannot contain a backslash (\\); write "/" instead.';
|
|
333
|
+
if (!/^https?:\/\/[^/]/i.test(value))
|
|
334
|
+
return 'Expected an absolute http(s) URL ("https://" and the host).';
|
|
167
335
|
// The engine decodes the userinfo into the Authorization header; a "%" that isn't an
|
|
168
336
|
// escape would fail there ("URI malformed") at request time. Reject it here.
|
|
169
337
|
for (const part of [url.username, url.password]) {
|
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Commands over the catalogue: datasets, their resources, publishers, themes
|
|
2
2
|
// and tags.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
|
-
import { facetLimitProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
5
|
-
import { action,
|
|
4
|
+
import { facetLimitProblem, fieldProblem, sortProblem, textProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
5
|
+
import { action, collectValues, once, parseBoundedInt, parseId, parseIntArg, parseOptionValue, parseText, } from "../shared.js";
|
|
6
|
+
import { renderJson } from "../output.js";
|
|
6
7
|
/**
|
|
7
8
|
* commander value-parser for a list --limit: 1 or more. CKAN reads `limit=0` as
|
|
8
9
|
* "no limit" and would send the whole list; leave --limit out for that.
|
|
@@ -23,12 +24,12 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
23
24
|
program
|
|
24
25
|
.command("search")
|
|
25
26
|
.description("Search datasets (Solr query syntax)")
|
|
26
|
-
.argument("[query]", "full-text query, e.g. elbe or title:haushalt",
|
|
27
|
+
.argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseText)
|
|
27
28
|
.option("--rows <n>", "max results (servers cap this, usually at 1000)", once(parseIntArg))
|
|
28
29
|
.option("--start <n>", "offset for paging", once(parseIntArg))
|
|
29
|
-
.option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue))
|
|
30
|
-
.option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)",
|
|
31
|
-
.option("--facet <field>", "count values of a field, e.g. res_format (repeatable)",
|
|
30
|
+
.option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue(sortProblem)))
|
|
31
|
+
.option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectValues(textProblem))
|
|
32
|
+
.option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectValues(fieldProblem))
|
|
32
33
|
.option("--facet-limit <n>", "max values per facet (default 50; -1 = all)", once(parseFacetLimit))
|
|
33
34
|
.action(action(deps, async ({ client, global, opts }, [query]) => {
|
|
34
35
|
renderJson(deps, global, await client.packageSearch({
|
|
@@ -44,28 +45,28 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
44
45
|
program
|
|
45
46
|
.command("package")
|
|
46
47
|
.description("Show one dataset by id or name")
|
|
47
|
-
.argument("<id>", "id or name",
|
|
48
|
+
.argument("<id>", "id or name", parseId)
|
|
48
49
|
.action(action(deps, async ({ client, global }, [id]) => {
|
|
49
50
|
renderJson(deps, global, await client.packageShow(id));
|
|
50
51
|
}));
|
|
51
52
|
program
|
|
52
53
|
.command("resource")
|
|
53
54
|
.description("Show one resource (distribution) by id")
|
|
54
|
-
.argument("<id>", "resource id",
|
|
55
|
+
.argument("<id>", "resource id", parseId)
|
|
55
56
|
.action(action(deps, async ({ client, global }, [id]) => {
|
|
56
57
|
renderJson(deps, global, await client.resourceShow(id));
|
|
57
58
|
}));
|
|
58
59
|
program
|
|
59
60
|
.command("organization")
|
|
60
61
|
.description("Show one organization (publisher) by id or name")
|
|
61
|
-
.argument("<id>", "id or name",
|
|
62
|
+
.argument("<id>", "id or name", parseId)
|
|
62
63
|
.action(action(deps, async ({ client, global }, [id]) => {
|
|
63
64
|
renderJson(deps, global, await client.organizationShow(id));
|
|
64
65
|
}));
|
|
65
66
|
program
|
|
66
67
|
.command("group")
|
|
67
68
|
.description("Show one group (theme/category) by id or name")
|
|
68
|
-
.argument("<id>", "id or name",
|
|
69
|
+
.argument("<id>", "id or name", parseId)
|
|
69
70
|
.action(action(deps, async ({ client, global }, [id]) => {
|
|
70
71
|
renderJson(deps, global, await client.groupShow(id));
|
|
71
72
|
}));
|
|
@@ -109,7 +110,7 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
109
110
|
program
|
|
110
111
|
.command("tags")
|
|
111
112
|
.description("List tags")
|
|
112
|
-
.option("--query <substring>", "only tags containing this substring", once(parseOptionValue))
|
|
113
|
+
.option("--query <substring>", "only tags containing this substring", once(parseOptionValue(textProblem)))
|
|
113
114
|
.action(action(deps, async ({ client, global, opts }) => {
|
|
114
115
|
renderJson(deps, global, await client.tagList({ query: opts["query"] }));
|
|
115
116
|
}));
|
|
@@ -1,40 +1,48 @@
|
|
|
1
1
|
// Commands about the portal itself rather than its datasets.
|
|
2
2
|
import { InvalidArgumentError } from "commander";
|
|
3
|
-
import { action,
|
|
3
|
+
import { action, toEngineOptions } from "../shared.js";
|
|
4
|
+
import { renderJson } from "../output.js";
|
|
4
5
|
import { checkPortals } from "@maschinenlesbar.org/ckan-client/internal/portals";
|
|
5
6
|
import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
7
|
+
import { normalizeToken, textProblem, tokenProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
8
|
+
import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
8
9
|
/**
|
|
9
|
-
* A typed value as a message quotes it:
|
|
10
|
-
* credentials are redacted (a cut could
|
|
11
|
-
* "@" the argv redaction keys on).
|
|
10
|
+
* A typed value as a message quotes it: visibly (`quoteValue`: invisible and control characters
|
|
11
|
+
* escaped, cut at MAX_MESSAGE_VALUE_LENGTH), after its credentials are redacted (a cut could
|
|
12
|
+
* otherwise leave part of a password without the "@" the argv redaction keys on).
|
|
12
13
|
*/
|
|
13
14
|
function shown(value) {
|
|
14
|
-
return
|
|
15
|
+
return quoteValue(redactUrl(value));
|
|
15
16
|
}
|
|
16
|
-
/**
|
|
17
|
+
/**
|
|
18
|
+
* commander accumulator for repeatable `key=value` pairs into a record. The library's rules,
|
|
19
|
+
* as a usage error: the name is a token (`tokenProblem`; sent trimmed, so ` q` and `q` are the
|
|
20
|
+
* same name, and giving it twice is refused), the value is text (`textProblem`: not blank, no
|
|
21
|
+
* control or invisible character).
|
|
22
|
+
*/
|
|
17
23
|
function collectKeyValue(value, previous = {}) {
|
|
18
24
|
const eq = value.indexOf("=");
|
|
19
25
|
// Throw commander's InvalidArgumentError (not a CkanError) so the failure is
|
|
20
26
|
// formatted as a usage error with `error:` prefix and help, consistent with
|
|
21
27
|
// other parse-time flag validation (e.g. `--rows abc`).
|
|
22
|
-
|
|
28
|
+
if (eq < 0)
|
|
29
|
+
throw new InvalidArgumentError(`Invalid --param ${shown(value)}. Expected key=value.`);
|
|
30
|
+
const key = value.slice(0, eq);
|
|
23
31
|
// A blank key (`=x`, ` =x`) names no parameter: the library refuses it too.
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
// Reject a duplicated key rather than silently overwriting the earlier value.
|
|
28
|
-
|
|
29
|
-
|
|
32
|
+
const keyProblem = tokenProblem(key);
|
|
33
|
+
if (keyProblem !== undefined)
|
|
34
|
+
throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the name: ${keyProblem}`);
|
|
35
|
+
// Reject a duplicated key (as it is sent, trimmed) rather than silently overwriting the earlier value.
|
|
36
|
+
const name = normalizeToken(key);
|
|
37
|
+
if (Object.keys(previous).some((k) => normalizeToken(k) === name)) {
|
|
38
|
+
throw new InvalidArgumentError(`Duplicate --param key ${shown(name)}.`);
|
|
30
39
|
}
|
|
31
40
|
// A blank value is refused by the library as well (CKAN reads an empty
|
|
32
|
-
// parameter as unset, so `--param q=` would run the action unfiltered)
|
|
33
|
-
// is a usage error that names the flag.
|
|
41
|
+
// parameter as unset, so `--param q=` would run the action unfiltered).
|
|
34
42
|
const paramValue = value.slice(eq + 1);
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
43
|
+
const valueProblem = textProblem(paramValue);
|
|
44
|
+
if (valueProblem !== undefined)
|
|
45
|
+
throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the value: ${valueProblem}`);
|
|
38
46
|
// A computed key in a literal is an own data property, even for `__proto__`.
|
|
39
47
|
return { ...previous, [key]: paramValue };
|
|
40
48
|
}
|
package/dist/src/cli/io.d.ts
CHANGED
|
@@ -58,4 +58,20 @@ export interface StdoutBacklog {
|
|
|
58
58
|
* process alive until then, so a held record is never lost at exit.
|
|
59
59
|
*/
|
|
60
60
|
export declare function stderrAfterStdout(stdout: StdoutBacklog, write: (text: string) => void): (text: string) => void;
|
|
61
|
+
/** What `stdoutWriter` needs of stdout. */
|
|
62
|
+
interface Stdout {
|
|
63
|
+
write(text: string): unknown;
|
|
64
|
+
emit(event: "error", err: Error): unknown;
|
|
65
|
+
readonly fd?: number;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* A writer for stdout that never drops a byte silently. When stdout is a regular file
|
|
69
|
+
* (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
|
|
70
|
+
* file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
|
|
71
|
+
* the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
|
|
72
|
+
* `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
|
|
73
|
+
* exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
|
|
74
|
+
*/
|
|
75
|
+
export declare function stdoutWriter(stdout?: Stdout, write?: (fd: number, data: Buffer) => void): (text: string) => void;
|
|
61
76
|
export declare const defaultIO: CliIO;
|
|
77
|
+
export {};
|
package/dist/src/cli/io.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// I/O seam for the CLI. Everything the CLI writes goes through a CliIO object so
|
|
2
2
|
// tests can capture output instead of hitting the real stdout/stderr, and the
|
|
3
3
|
// environment is injected so tests never depend on the real process.env.
|
|
4
|
+
import { fstatSync, writeSync } from "node:fs";
|
|
4
5
|
import { createLogger } from "./log.js";
|
|
5
6
|
/** The deps' logger, or one that writes text records through `io.err`. */
|
|
6
7
|
export function logOf(deps) {
|
|
@@ -78,8 +79,61 @@ export function stderrAfterStdout(stdout, write) {
|
|
|
78
79
|
};
|
|
79
80
|
}
|
|
80
81
|
const stderrLine = stderrAfterStdout(process.stdout, (text) => process.stderr.write(text + "\n"));
|
|
82
|
+
/** A short pause for a non-blocking descriptor that is full (EAGAIN), without a busy loop. */
|
|
83
|
+
const pause = new Int32Array(new SharedArrayBuffer(4));
|
|
84
|
+
/**
|
|
85
|
+
* Write all of `data` to `fd`, or throw. `writeSync` may write less than asked (a file-size
|
|
86
|
+
* limit, a full disk): the rest is written in a loop, and the call that cannot write throws
|
|
87
|
+
* (EFBIG, ENOSPC). A descriptor the shell made non-blocking (a pipe through `/dev/fd/N`) answers
|
|
88
|
+
* EAGAIN while the reader is behind: wait a moment and go on.
|
|
89
|
+
*/
|
|
90
|
+
function writeAll(fd, data) {
|
|
91
|
+
for (let at = 0; at < data.length;) {
|
|
92
|
+
try {
|
|
93
|
+
at += writeSync(fd, data, at);
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
if (err.code !== "EAGAIN")
|
|
97
|
+
throw err;
|
|
98
|
+
Atomics.wait(pause, 0, 0, 5);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* A writer for stdout that never drops a byte silently. When stdout is a regular file
|
|
104
|
+
* (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
|
|
105
|
+
* file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
|
|
106
|
+
* the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
|
|
107
|
+
* `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
|
|
108
|
+
* exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
|
|
109
|
+
*/
|
|
110
|
+
export function stdoutWriter(stdout = process.stdout, write = writeAll) {
|
|
111
|
+
let toFile;
|
|
112
|
+
return (text) => {
|
|
113
|
+
toFile ??= stdout.fd !== undefined && isRegularFile(stdout.fd);
|
|
114
|
+
if (!toFile) {
|
|
115
|
+
stdout.write(text);
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
try {
|
|
119
|
+
write(stdout.fd, Buffer.from(text, "utf8"));
|
|
120
|
+
}
|
|
121
|
+
catch (err) {
|
|
122
|
+
stdout.emit("error", err instanceof Error ? err : new Error(String(err)));
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
function isRegularFile(fd) {
|
|
127
|
+
try {
|
|
128
|
+
return fstatSync(fd).isFile();
|
|
129
|
+
}
|
|
130
|
+
catch {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const stdoutText = stdoutWriter();
|
|
81
135
|
export const defaultIO = {
|
|
82
|
-
out: (text) =>
|
|
136
|
+
out: (text) => stdoutText(text + "\n"),
|
|
83
137
|
// A record never lands inside the data when both streams share a pipe.
|
|
84
138
|
err: stderrLine,
|
|
85
139
|
};
|