@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/README.md +21 -7
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
  15. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
  16. package/dist/src/cli/commands/catalogue.js +12 -11
  17. package/dist/src/cli/commands/portal.js +28 -20
  18. package/dist/src/cli/io.d.ts +16 -0
  19. package/dist/src/cli/io.js +55 -1
  20. package/dist/src/cli/output.d.ts +26 -0
  21. package/dist/src/cli/output.js +151 -0
  22. package/dist/src/cli/run.d.ts +0 -7
  23. package/dist/src/cli/run.js +80 -17
  24. package/dist/src/cli/shared.d.ts +14 -25
  25. package/dist/src/cli/shared.js +40 -102
  26. package/package.json +1 -1
@@ -7,7 +7,11 @@ export type Problem<T = unknown> = (value: T) => string | undefined;
7
7
  * rather than throwing synchronously.
8
8
  */
9
9
  export declare function assertValid<T>(name: string, value: T, problem: Problem<T>): T;
10
- /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
10
+ /**
11
+ * True when nothing in `value` would match anything: it holds only whitespace, control and
12
+ * format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
13
+ * empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
14
+ */
11
15
  export declare function isBlank(value: string): boolean;
12
16
  /**
13
17
  * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
@@ -15,25 +19,76 @@ export declare function isBlank(value: string): boolean;
15
19
  * silently return the unfiltered result. Values that are not strings pass.
16
20
  */
17
21
  export declare function blankProblem(value: unknown): string | undefined;
18
- /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
19
- export declare function textProblem(value: unknown): string | undefined;
22
+ /**
23
+ * Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
24
+ * an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
25
+ * would match nothing or everything:
26
+ * - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
27
+ * - a lone surrogate (half of a character, sent as U+FFFD);
28
+ * - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
29
+ * hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
30
+ * nothing, or something else (Solr splits a word at a zero-width space).
31
+ * The reason quotes the value with every invisible character escaped (`quoteValue`).
32
+ */
33
+ export declare function textProblem(value: string): string | undefined;
20
34
  /**
21
35
  * A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
22
36
  * with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
23
37
  * "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
24
- * composed form matches). Inner whitespace is kept. Used for the search text (`q`) and
25
- * the tag query, never for ids, Solr filters (`fq`), sort expressions or field names.
38
+ * composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
39
+ * query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
26
40
  * NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
27
41
  * meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
28
42
  * forms) that a phrase or a wildcard may mean literally.
29
43
  */
30
44
  export declare function normalizeText(value: string): string;
31
- /** A list of non-blank strings (filter queries, facet fields). */
32
- export declare function textListProblem(value: unknown): string | undefined;
45
+ /**
46
+ * Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
47
+ * field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
48
+ * with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
49
+ * only invisible characters or combining marks), no lone surrogate, and no control or format
50
+ * character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
51
+ * character in an id or a field name only ever makes it match nothing. The reason quotes the
52
+ * value visibly.
53
+ */
54
+ export declare function tokenProblem(value: string): string | undefined;
55
+ /** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
56
+ export declare function normalizeToken(value: string): string;
57
+ /**
58
+ * Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
59
+ * (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
60
+ * no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
61
+ * hold a `/`; a value with one is a pasted URL or path, not an id.
62
+ */
63
+ export declare function idProblem(value: string): string | undefined;
64
+ /**
65
+ * Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
66
+ * {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
67
+ * `extras_registerobject_type`). A field name with a space or another character names no field,
68
+ * and CKAN answers with an empty facet and exit 0.
69
+ */
70
+ export declare function fieldProblem(value: string): string | undefined;
71
+ /**
72
+ * Why `value` cannot be sent as a sort expression, or undefined when it can: the
73
+ * {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
74
+ * and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
75
+ * refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
76
+ * not caught: CKAN sorts by its default then, as documented.
77
+ */
78
+ export declare function sortProblem(value: string): string | undefined;
79
+ /**
80
+ * Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
81
+ * item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
82
+ * default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
83
+ */
84
+ export declare function listProblem(items: readonly string[], itemProblem?: Problem<string>): string | undefined;
33
85
  /**
34
86
  * A value the query-string builder can send: a string, a finite number, a boolean
35
- * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
36
- * function or `NaN` would go out as `[object Object]` or `NaN`.
87
+ * or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
88
+ * object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
89
+ * least one value and no `null`/`undefined` entry: the query builder drops those, so
90
+ * `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
91
+ * list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
37
92
  */
38
93
  export declare function queryValueProblem(value: unknown): string | undefined;
39
94
  /**
@@ -62,13 +117,17 @@ export declare function headerValueProblem(value: string): string | undefined;
62
117
  *
63
118
  * - not blank;
64
119
  * - no surrounding or embedded whitespace and no control characters: `new URL`
65
- * trims and drops tab/newline silently, but the client glues the value into
66
- * every request URL as given, so a trailing space would request another path
67
- * (`/ckan%20/api/3/...`) or reach a custom transport raw;
120
+ * trims and drops tab/newline silently, so the URL requested would not be the one
121
+ * typed (a trailing space would request another path, `/ckan%20/api/3/...`);
68
122
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
69
- * transport);
123
+ * transport); another scheme is named only when the value spells `<scheme>://`, since
124
+ * `user:password@host` parses with the user name as its scheme;
70
125
  * - no query string or fragment: the API path is appended, so it would land in
71
126
  * front of `/api/3/action`;
127
+ * - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
128
+ * `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
129
+ * any slashes after the scheme, so text-based code would read another host or no userinfo
130
+ * (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
72
131
  * - a `%` in the user name or password must start a valid escape (`%25` for a
73
132
  * literal one): the engine decodes the userinfo for the Authorization header.
74
133
  *
@@ -2,7 +2,7 @@
2
2
  // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
3
  // enforce them with `assertValid` before any request; the CLI's value parsers call
4
4
  // the same functions, so a rule is written once and the CLI and the library agree.
5
- import { CkanValidationError, cutForMessage } from "./errors.js";
5
+ import { CkanValidationError, cutForMessage, quoteValue } from "./errors.js";
6
6
  /**
7
7
  * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
8
  * something wrong with `value`; otherwise return `value` unchanged. A method that
@@ -15,9 +15,17 @@ export function assertValid(name, value, problem) {
15
15
  throw new CkanValidationError(`Invalid ${name}: ${reason}`);
16
16
  return value;
17
17
  }
18
- /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
18
+ /** Characters that never make a value non-blank: whitespace, controls, format characters, combining marks. */
19
+ const INVISIBLE = /[\s\p{Cc}\p{Cf}\p{M}]/gu;
20
+ /** A lone surrogate: half of a character, which the query string would send as U+FFFD. */
21
+ const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
22
+ /**
23
+ * True when nothing in `value` would match anything: it holds only whitespace, control and
24
+ * format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
25
+ * empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
26
+ */
19
27
  export function isBlank(value) {
20
- return value.trim() === "";
28
+ return value.replace(INVISIBLE, "") === "";
21
29
  }
22
30
  /**
23
31
  * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
@@ -30,18 +38,45 @@ export function blankProblem(value) {
30
38
  return "Expected a non-empty value.";
31
39
  return undefined;
32
40
  }
33
- /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
41
+ /** `U+200B` for the first character of `text` that `pattern` matches. */
42
+ function codePointOf(text, pattern) {
43
+ const ch = pattern.exec(text)?.[0] ?? "";
44
+ return `U+${(ch.codePointAt(0) ?? 0).toString(16).toUpperCase().padStart(4, "0")}`;
45
+ }
46
+ /** A control character inside a text value: C0 and C1, but not TAB, LF or CR (whitespace in a pasted phrase). */
47
+ const TEXT_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/;
48
+ /** A format character inside a text value, but not ZWNJ/ZWJ (U+200C/U+200D), which some scripts and emoji need. */
49
+ const TEXT_FORMAT = /(?![\u200c\u200d])\p{Cf}/u;
50
+ /**
51
+ * Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
52
+ * an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
53
+ * would match nothing or everything:
54
+ * - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
55
+ * - a lone surrogate (half of a character, sent as U+FFFD);
56
+ * - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
57
+ * hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
58
+ * nothing, or something else (Solr splits a word at a zero-width space).
59
+ * The reason quotes the value with every invisible character escaped (`quoteValue`).
60
+ */
34
61
  export function textProblem(value) {
35
62
  if (typeof value !== "string")
36
63
  return "Expected a string.";
37
- return blankProblem(value);
64
+ if (isBlank(value))
65
+ return "Expected a non-empty value.";
66
+ if (LONE_SURROGATE.test(value))
67
+ return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
68
+ if (TEXT_CONTROL.test(value))
69
+ return `${quoteValue(value)} contains a control character (${codePointOf(value, TEXT_CONTROL)}).`;
70
+ if (TEXT_FORMAT.test(value))
71
+ return `${quoteValue(value)} contains an invisible character (${codePointOf(value, TEXT_FORMAT)}).`;
72
+ return undefined;
38
73
  }
39
74
  /**
40
75
  * A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
41
76
  * with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
42
77
  * "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
43
- * composed form matches). Inner whitespace is kept. Used for the search text (`q`) and
44
- * the tag query, never for ids, Solr filters (`fq`), sort expressions or field names.
78
+ * composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
79
+ * query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
45
80
  * NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
46
81
  * meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
47
82
  * forms) that a phrase or a wildcard may mean literally.
@@ -49,29 +84,145 @@ export function textProblem(value) {
49
84
  export function normalizeText(value) {
50
85
  return value.trim().normalize("NFC");
51
86
  }
52
- /** A list of non-blank strings (filter queries, facet fields). */
53
- export function textListProblem(value) {
54
- if (!Array.isArray(value))
55
- return "Expected a list of strings.";
56
- if (value.some((v) => typeof v !== "string"))
87
+ /** A control or format character in a token (after trimming): every Cc and every Cf. */
88
+ const TOKEN_INVISIBLE = /[\p{Cc}\p{Cf}]/u;
89
+ /**
90
+ * Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
91
+ * field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
92
+ * with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
93
+ * only invisible characters or combining marks), no lone surrogate, and no control or format
94
+ * character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
95
+ * character in an id or a field name only ever makes it match nothing. The reason quotes the
96
+ * value visibly.
97
+ */
98
+ export function tokenProblem(value) {
99
+ if (typeof value !== "string")
100
+ return "Expected a string.";
101
+ if (isBlank(value))
102
+ return "Expected a non-empty value.";
103
+ const token = value.trim();
104
+ if (LONE_SURROGATE.test(token))
105
+ return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
106
+ if (TOKEN_INVISIBLE.test(token)) {
107
+ return `${quoteValue(value)} contains an invisible or control character (${codePointOf(token, TOKEN_INVISIBLE)}).`;
108
+ }
109
+ return undefined;
110
+ }
111
+ /** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
112
+ export function normalizeToken(value) {
113
+ return value.trim().normalize("NFC");
114
+ }
115
+ /**
116
+ * Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
117
+ * (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
118
+ * no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
119
+ * hold a `/`; a value with one is a pasted URL or path, not an id.
120
+ */
121
+ export function idProblem(value) {
122
+ const problem = tokenProblem(value);
123
+ if (problem !== undefined)
124
+ return problem;
125
+ const id = value.trim();
126
+ if (id.includes("/"))
127
+ return `${quoteValue(value)} is not an id or name: it contains "/".`;
128
+ if (id === "." || id === "..")
129
+ return '"." and ".." are not ids.';
130
+ return undefined;
131
+ }
132
+ /**
133
+ * Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
134
+ * {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
135
+ * `extras_registerobject_type`). A field name with a space or another character names no field,
136
+ * and CKAN answers with an empty facet and exit 0.
137
+ */
138
+ export function fieldProblem(value) {
139
+ const problem = idProblem(value);
140
+ if (problem !== undefined)
141
+ return problem;
142
+ if (!/^[\w.-]+$/.test(value.trim()))
143
+ return `${quoteValue(value)} is not a Solr field name (letters, digits, _ . -), e.g. res_format.`;
144
+ return undefined;
145
+ }
146
+ /**
147
+ * Why `value` cannot be sent as a sort expression, or undefined when it can: the
148
+ * {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
149
+ * and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
150
+ * refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
151
+ * not caught: CKAN sorts by its default then, as documented.
152
+ */
153
+ export function sortProblem(value) {
154
+ const problem = tokenProblem(value);
155
+ if (problem !== undefined)
156
+ return problem;
157
+ // Split at the commas outside parentheses: a function sort (`div(a,b) desc`) holds commas.
158
+ const criteria = [];
159
+ let depth = 0;
160
+ let from = 0;
161
+ const sort = value.trim();
162
+ for (let i = 0; i < sort.length; i++) {
163
+ const c = sort[i];
164
+ if (c === "(")
165
+ depth++;
166
+ else if (c === ")")
167
+ depth--;
168
+ else if (c === "," && depth === 0) {
169
+ criteria.push(sort.slice(from, i));
170
+ from = i + 1;
171
+ }
172
+ }
173
+ criteria.push(sort.slice(from));
174
+ if (criteria.some((criterion) => !/^\s*\S.*\s(asc|desc)\s*$/i.test(criterion))) {
175
+ return `${quoteValue(value)} is not a sort expression: expected "<field> asc|desc[, <field> asc|desc]", e.g. "metadata_modified desc".`;
176
+ }
177
+ return undefined;
178
+ }
179
+ /**
180
+ * Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
181
+ * item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
182
+ * default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
183
+ */
184
+ export function listProblem(items, itemProblem = tokenProblem) {
185
+ if (!Array.isArray(items))
57
186
  return "Expected a list of strings.";
58
- return blankProblem(value);
187
+ if (items.length === 0)
188
+ return "Expected at least one value.";
189
+ for (let i = 0; i < items.length; i++) {
190
+ const problem = itemProblem(items[i]);
191
+ if (problem !== undefined)
192
+ return items.length === 1 ? problem : `item ${i + 1} of ${items.length}: ${problem}`;
193
+ }
194
+ return undefined;
59
195
  }
60
196
  /**
61
197
  * A value the query-string builder can send: a string, a finite number, a boolean
62
- * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
63
- * function or `NaN` would go out as `[object Object]` or `NaN`.
198
+ * or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
199
+ * object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
200
+ * least one value and no `null`/`undefined` entry: the query builder drops those, so
201
+ * `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
202
+ * list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
64
203
  */
65
204
  export function queryValueProblem(value) {
205
+ if (Array.isArray(value)) {
206
+ if (value.length === 0)
207
+ return "Expected at least one value; leave the parameter out instead.";
208
+ const missing = value.findIndex((v) => v === null || v === undefined);
209
+ if (missing >= 0) {
210
+ return `item ${missing + 1} of ${value.length} is ${String(value[missing])}; a list holds values only (leave the parameter out instead).`;
211
+ }
212
+ }
66
213
  const ok = (v) => v === null ||
67
214
  v === undefined ||
68
215
  typeof v === "string" ||
69
216
  typeof v === "boolean" ||
70
217
  (typeof v === "number" && Number.isFinite(v)) ||
71
218
  (v instanceof Date && !Number.isNaN(v.getTime()));
72
- if (Array.isArray(value) ? value.every(ok) : ok(value))
73
- return blankProblem(value);
74
- return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
219
+ if (!(Array.isArray(value) ? value.every(ok) : ok(value))) {
220
+ return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
221
+ }
222
+ // A string goes out as text (normalizeText): the text rules apply to it and to each list entry.
223
+ const strings = (Array.isArray(value) ? value : [value]).filter((v) => typeof v === "string");
224
+ const problem = listProblem(strings, textProblem);
225
+ return strings.length === 0 ? undefined : problem;
75
226
  }
76
227
  /**
77
228
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
@@ -127,13 +278,17 @@ export function headerValueProblem(value) {
127
278
  *
128
279
  * - not blank;
129
280
  * - no surrounding or embedded whitespace and no control characters: `new URL`
130
- * trims and drops tab/newline silently, but the client glues the value into
131
- * every request URL as given, so a trailing space would request another path
132
- * (`/ckan%20/api/3/...`) or reach a custom transport raw;
281
+ * trims and drops tab/newline silently, so the URL requested would not be the one
282
+ * typed (a trailing space would request another path, `/ckan%20/api/3/...`);
133
283
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
134
- * transport);
284
+ * transport); another scheme is named only when the value spells `<scheme>://`, since
285
+ * `user:password@host` parses with the user name as its scheme;
135
286
  * - no query string or fragment: the API path is appended, so it would land in
136
287
  * front of `/api/3/action`;
288
+ * - no backslash, and `http://` or `https://` followed by the host (not `http:host`,
289
+ * `http:/host` or `http:///host`): WHATWG ends an http(s) host at a backslash and skips
290
+ * any slashes after the scheme, so text-based code would read another host or no userinfo
291
+ * (`https://u:pw@a.example\@b.example` sent `u:pw` to b.example);
137
292
  * - a `%` in the user name or password must start a valid escape (`%25` for a
138
293
  * literal one): the engine decodes the userinfo for the Authorization header.
139
294
  *
@@ -160,10 +315,23 @@ export function baseUrlProblem(value) {
160
315
  return "Expected an absolute http(s) URL.";
161
316
  }
162
317
  if (url.protocol !== "http:" && url.protocol !== "https:") {
318
+ // `user:password@host` without a scheme parses as a URL whose scheme is the user name
319
+ // ("t0kens3cret:"), lower-cased, which no exact-string redaction can match: name a scheme
320
+ // only when the value spells one, `<scheme>://`.
321
+ if (!/^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value))
322
+ return "Expected an absolute http(s) URL.";
163
323
  return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
164
324
  }
165
325
  if (/[?#]/.test(value))
166
326
  return "Expected a site URL without a query string or fragment.";
327
+ // WHATWG reads a "\" in an http(s) URL as "/", so it ends the host, and it skips any number of
328
+ // slashes after "http:": `http://u:pw@a.example\@b.example` is a.example to the parser and
329
+ // was b.example to text-based code, and `http:u:pw@host` hid the userinfo from the redaction.
330
+ // Every reader of the base URL has to agree where its host is.
331
+ if (value.includes("\\"))
332
+ return 'A base URL cannot contain a backslash (\\); write "/" instead.';
333
+ if (!/^https?:\/\/[^/]/i.test(value))
334
+ return 'Expected an absolute http(s) URL ("https://" and the host).';
167
335
  // The engine decodes the userinfo into the Authorization header; a "%" that isn't an
168
336
  // escape would fail there ("URI malformed") at request time. Reject it here.
169
337
  for (const part of [url.username, url.password]) {
@@ -1,8 +1,9 @@
1
1
  // Commands over the catalogue: datasets, their resources, publishers, themes
2
2
  // and tags.
3
3
  import { InvalidArgumentError } from "commander";
4
- import { facetLimitProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
5
- import { action, collectNonEmpty, once, parseBoundedInt, parseIntArg, parseNonEmpty, parseOptionValue, renderJson, } from "../shared.js";
4
+ import { facetLimitProblem, fieldProblem, sortProblem, textProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
5
+ import { action, collectValues, once, parseBoundedInt, parseId, parseIntArg, parseOptionValue, parseText, } from "../shared.js";
6
+ import { renderJson } from "../output.js";
6
7
  /**
7
8
  * commander value-parser for a list --limit: 1 or more. CKAN reads `limit=0` as
8
9
  * "no limit" and would send the whole list; leave --limit out for that.
@@ -23,12 +24,12 @@ export function registerCatalogueCommands(program, deps) {
23
24
  program
24
25
  .command("search")
25
26
  .description("Search datasets (Solr query syntax)")
26
- .argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseNonEmpty)
27
+ .argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseText)
27
28
  .option("--rows <n>", "max results (servers cap this, usually at 1000)", once(parseIntArg))
28
29
  .option("--start <n>", "offset for paging", once(parseIntArg))
29
- .option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue))
30
- .option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectNonEmpty)
31
- .option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectNonEmpty)
30
+ .option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue(sortProblem)))
31
+ .option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectValues(textProblem))
32
+ .option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectValues(fieldProblem))
32
33
  .option("--facet-limit <n>", "max values per facet (default 50; -1 = all)", once(parseFacetLimit))
33
34
  .action(action(deps, async ({ client, global, opts }, [query]) => {
34
35
  renderJson(deps, global, await client.packageSearch({
@@ -44,28 +45,28 @@ export function registerCatalogueCommands(program, deps) {
44
45
  program
45
46
  .command("package")
46
47
  .description("Show one dataset by id or name")
47
- .argument("<id>", "id or name", parseNonEmpty)
48
+ .argument("<id>", "id or name", parseId)
48
49
  .action(action(deps, async ({ client, global }, [id]) => {
49
50
  renderJson(deps, global, await client.packageShow(id));
50
51
  }));
51
52
  program
52
53
  .command("resource")
53
54
  .description("Show one resource (distribution) by id")
54
- .argument("<id>", "resource id", parseNonEmpty)
55
+ .argument("<id>", "resource id", parseId)
55
56
  .action(action(deps, async ({ client, global }, [id]) => {
56
57
  renderJson(deps, global, await client.resourceShow(id));
57
58
  }));
58
59
  program
59
60
  .command("organization")
60
61
  .description("Show one organization (publisher) by id or name")
61
- .argument("<id>", "id or name", parseNonEmpty)
62
+ .argument("<id>", "id or name", parseId)
62
63
  .action(action(deps, async ({ client, global }, [id]) => {
63
64
  renderJson(deps, global, await client.organizationShow(id));
64
65
  }));
65
66
  program
66
67
  .command("group")
67
68
  .description("Show one group (theme/category) by id or name")
68
- .argument("<id>", "id or name", parseNonEmpty)
69
+ .argument("<id>", "id or name", parseId)
69
70
  .action(action(deps, async ({ client, global }, [id]) => {
70
71
  renderJson(deps, global, await client.groupShow(id));
71
72
  }));
@@ -109,7 +110,7 @@ export function registerCatalogueCommands(program, deps) {
109
110
  program
110
111
  .command("tags")
111
112
  .description("List tags")
112
- .option("--query <substring>", "only tags containing this substring", once(parseOptionValue))
113
+ .option("--query <substring>", "only tags containing this substring", once(parseOptionValue(textProblem)))
113
114
  .action(action(deps, async ({ client, global, opts }) => {
114
115
  renderJson(deps, global, await client.tagList({ query: opts["query"] }));
115
116
  }));
@@ -1,40 +1,48 @@
1
1
  // Commands about the portal itself rather than its datasets.
2
2
  import { InvalidArgumentError } from "commander";
3
- import { action, renderJson, toEngineOptions } from "../shared.js";
3
+ import { action, toEngineOptions } from "../shared.js";
4
+ import { renderJson } from "../output.js";
4
5
  import { checkPortals } from "@maschinenlesbar.org/ckan-client/internal/portals";
5
6
  import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
6
- import { isBlank } from "@maschinenlesbar.org/ckan-client/internal/validate";
7
- import { cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
7
+ import { normalizeToken, textProblem, tokenProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
8
+ import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
8
9
  /**
9
- * A typed value as a message quotes it: cut at MAX_MESSAGE_VALUE_LENGTH, after its
10
- * credentials are redacted (a cut could otherwise leave part of a password without the
11
- * "@" the argv redaction keys on).
10
+ * A typed value as a message quotes it: visibly (`quoteValue`: invisible and control characters
11
+ * escaped, cut at MAX_MESSAGE_VALUE_LENGTH), after its credentials are redacted (a cut could
12
+ * otherwise leave part of a password without the "@" the argv redaction keys on).
12
13
  */
13
14
  function shown(value) {
14
- return cutForMessage(redactUrl(value));
15
+ return quoteValue(redactUrl(value));
15
16
  }
16
- /** commander accumulator for repeatable `key=value` pairs into a record. */
17
+ /**
18
+ * commander accumulator for repeatable `key=value` pairs into a record. The library's rules,
19
+ * as a usage error: the name is a token (`tokenProblem`; sent trimmed, so ` q` and `q` are the
20
+ * same name, and giving it twice is refused), the value is text (`textProblem`: not blank, no
21
+ * control or invisible character).
22
+ */
17
23
  function collectKeyValue(value, previous = {}) {
18
24
  const eq = value.indexOf("=");
19
25
  // Throw commander's InvalidArgumentError (not a CkanError) so the failure is
20
26
  // formatted as a usage error with `error:` prefix and help, consistent with
21
27
  // other parse-time flag validation (e.g. `--rows abc`).
22
- const key = value.slice(0, Math.max(eq, 0));
28
+ if (eq < 0)
29
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}. Expected key=value.`);
30
+ const key = value.slice(0, eq);
23
31
  // A blank key (`=x`, ` =x`) names no parameter: the library refuses it too.
24
- if (eq < 0 || isBlank(key)) {
25
- throw new InvalidArgumentError(`Invalid --param "${shown(value)}". Expected key=value.`);
26
- }
27
- // Reject a duplicated key rather than silently overwriting the earlier value.
28
- if (Object.prototype.hasOwnProperty.call(previous, key)) {
29
- throw new InvalidArgumentError(`Duplicate --param key "${shown(key)}".`);
32
+ const keyProblem = tokenProblem(key);
33
+ if (keyProblem !== undefined)
34
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the name: ${keyProblem}`);
35
+ // Reject a duplicated key (as it is sent, trimmed) rather than silently overwriting the earlier value.
36
+ const name = normalizeToken(key);
37
+ if (Object.keys(previous).some((k) => normalizeToken(k) === name)) {
38
+ throw new InvalidArgumentError(`Duplicate --param key ${shown(name)}.`);
30
39
  }
31
40
  // A blank value is refused by the library as well (CKAN reads an empty
32
- // parameter as unset, so `--param q=` would run the action unfiltered); here it
33
- // is a usage error that names the flag.
41
+ // parameter as unset, so `--param q=` would run the action unfiltered).
34
42
  const paramValue = value.slice(eq + 1);
35
- if (isBlank(paramValue)) {
36
- throw new InvalidArgumentError(`Invalid --param "${shown(value)}". The value must not be blank.`);
37
- }
43
+ const valueProblem = textProblem(paramValue);
44
+ if (valueProblem !== undefined)
45
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the value: ${valueProblem}`);
38
46
  // A computed key in a literal is an own data property, even for `__proto__`.
39
47
  return { ...previous, [key]: paramValue };
40
48
  }
@@ -58,4 +58,20 @@ export interface StdoutBacklog {
58
58
  * process alive until then, so a held record is never lost at exit.
59
59
  */
60
60
  export declare function stderrAfterStdout(stdout: StdoutBacklog, write: (text: string) => void): (text: string) => void;
61
+ /** What `stdoutWriter` needs of stdout. */
62
+ interface Stdout {
63
+ write(text: string): unknown;
64
+ emit(event: "error", err: Error): unknown;
65
+ readonly fd?: number;
66
+ }
67
+ /**
68
+ * A writer for stdout that never drops a byte silently. When stdout is a regular file
69
+ * (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
70
+ * file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
71
+ * the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
72
+ * `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
73
+ * exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
74
+ */
75
+ export declare function stdoutWriter(stdout?: Stdout, write?: (fd: number, data: Buffer) => void): (text: string) => void;
61
76
  export declare const defaultIO: CliIO;
77
+ export {};
@@ -1,6 +1,7 @@
1
1
  // I/O seam for the CLI. Everything the CLI writes goes through a CliIO object so
2
2
  // tests can capture output instead of hitting the real stdout/stderr, and the
3
3
  // environment is injected so tests never depend on the real process.env.
4
+ import { fstatSync, writeSync } from "node:fs";
4
5
  import { createLogger } from "./log.js";
5
6
  /** The deps' logger, or one that writes text records through `io.err`. */
6
7
  export function logOf(deps) {
@@ -78,8 +79,61 @@ export function stderrAfterStdout(stdout, write) {
78
79
  };
79
80
  }
80
81
  const stderrLine = stderrAfterStdout(process.stdout, (text) => process.stderr.write(text + "\n"));
82
+ /** A short pause for a non-blocking descriptor that is full (EAGAIN), without a busy loop. */
83
+ const pause = new Int32Array(new SharedArrayBuffer(4));
84
+ /**
85
+ * Write all of `data` to `fd`, or throw. `writeSync` may write less than asked (a file-size
86
+ * limit, a full disk): the rest is written in a loop, and the call that cannot write throws
87
+ * (EFBIG, ENOSPC). A descriptor the shell made non-blocking (a pipe through `/dev/fd/N`) answers
88
+ * EAGAIN while the reader is behind: wait a moment and go on.
89
+ */
90
+ function writeAll(fd, data) {
91
+ for (let at = 0; at < data.length;) {
92
+ try {
93
+ at += writeSync(fd, data, at);
94
+ }
95
+ catch (err) {
96
+ if (err.code !== "EAGAIN")
97
+ throw err;
98
+ Atomics.wait(pause, 0, 0, 5);
99
+ }
100
+ }
101
+ }
102
+ /**
103
+ * A writer for stdout that never drops a byte silently. When stdout is a regular file
104
+ * (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
105
+ * file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
106
+ * the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
107
+ * `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
108
+ * exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
109
+ */
110
+ export function stdoutWriter(stdout = process.stdout, write = writeAll) {
111
+ let toFile;
112
+ return (text) => {
113
+ toFile ??= stdout.fd !== undefined && isRegularFile(stdout.fd);
114
+ if (!toFile) {
115
+ stdout.write(text);
116
+ return;
117
+ }
118
+ try {
119
+ write(stdout.fd, Buffer.from(text, "utf8"));
120
+ }
121
+ catch (err) {
122
+ stdout.emit("error", err instanceof Error ? err : new Error(String(err)));
123
+ }
124
+ };
125
+ }
126
+ function isRegularFile(fd) {
127
+ try {
128
+ return fstatSync(fd).isFile();
129
+ }
130
+ catch {
131
+ return false;
132
+ }
133
+ }
134
+ const stdoutText = stdoutWriter();
81
135
  export const defaultIO = {
82
- out: (text) => process.stdout.write(text + "\n"),
136
+ out: (text) => stdoutText(text + "\n"),
83
137
  // A record never lands inside the data when both streams share a pipe.
84
138
  err: stderrLine,
85
139
  };