@maschinenlesbar.org/ckan-cli 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. package/README.md +25 -9
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +17 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +102 -49
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +90 -11
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +337 -21
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +16 -1
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +33 -1
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +10 -7
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/types.d.ts +11 -11
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +73 -7
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +180 -16
  15. package/dist/src/cli/commands/catalogue.js +12 -11
  16. package/dist/src/cli/commands/portal.js +28 -20
  17. package/dist/src/cli/io.d.ts +16 -0
  18. package/dist/src/cli/io.js +55 -1
  19. package/dist/src/cli/output.d.ts +26 -0
  20. package/dist/src/cli/output.js +151 -0
  21. package/dist/src/cli/run.js +76 -6
  22. package/dist/src/cli/shared.d.ts +14 -25
  23. package/dist/src/cli/shared.js +40 -72
  24. package/package.json +1 -1
@@ -35,22 +35,22 @@ export interface PackageSearchResult {
35
35
  /** Parameters for `package_search`. */
36
36
  export interface PackageSearchParams {
37
37
  /** Solr query string, e.g. `elbe` or `title:Haushalt`. */
38
- q?: string;
38
+ q?: string | undefined;
39
39
  /** Filter queries, e.g. `["organization:allris"]`. All must match. */
40
- fq?: string[];
40
+ fq?: string[] | undefined;
41
41
  /** Page size. CKAN caps it server-side (`ckan.search.rows_max`, 1000 by default). */
42
- rows?: number;
42
+ rows?: number | undefined;
43
43
  /** Offset of the first result, for paging. */
44
- start?: number;
44
+ start?: number | undefined;
45
45
  /** e.g. `"metadata_modified desc"`. */
46
- sort?: string;
46
+ sort?: string | undefined;
47
47
  /**
48
48
  * Facet fields to count, e.g. `["organization", "res_format"]`. Counts come
49
49
  * back in `search_facets`.
50
50
  */
51
- facet_field?: string[];
51
+ facet_field?: string[] | undefined;
52
52
  /** Maximum number of values returned per facet (CKAN's default is 50; -1 = all). */
53
- facet_limit?: number;
53
+ facet_limit?: number | undefined;
54
54
  }
55
55
  /** An organization (a data publisher) or a group (a theme/category). */
56
56
  export type Organization = JsonObject;
@@ -64,18 +64,18 @@ export interface ListParams {
64
64
  * is refused: CKAN reads `limit=0` as "no limit", which the `all_fields` pager
65
65
  * would read as "no entries".
66
66
  */
67
- limit?: number;
68
- offset?: number;
67
+ limit?: number | undefined;
68
+ offset?: number | undefined;
69
69
  }
70
70
  /** Parameters for `organization_list` / `group_list`. */
71
71
  export interface GroupListParams extends ListParams {
72
72
  /** Return full objects instead of just names. */
73
- all_fields?: boolean;
73
+ all_fields?: boolean | undefined;
74
74
  }
75
75
  /** Parameters for `tag_list`. */
76
76
  export interface TagListParams {
77
77
  /** Only tags containing this substring. */
78
- query?: string;
78
+ query?: string | undefined;
79
79
  }
80
80
  /** A licence a portal offers (`license_list`). */
81
81
  export type License = JsonObject;
@@ -7,7 +7,11 @@ export type Problem<T = unknown> = (value: T) => string | undefined;
7
7
  * rather than throwing synchronously.
8
8
  */
9
9
  export declare function assertValid<T>(name: string, value: T, problem: Problem<T>): T;
10
- /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
10
+ /**
11
+ * True when nothing in `value` would match anything: it holds only whitespace, control and
12
+ * format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
13
+ * empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
14
+ */
11
15
  export declare function isBlank(value: string): boolean;
12
16
  /**
13
17
  * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
@@ -15,14 +19,76 @@ export declare function isBlank(value: string): boolean;
15
19
  * silently return the unfiltered result. Values that are not strings pass.
16
20
  */
17
21
  export declare function blankProblem(value: unknown): string | undefined;
18
- /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
19
- export declare function textProblem(value: unknown): string | undefined;
20
- /** A list of non-blank strings (filter queries, facet fields). */
21
- export declare function textListProblem(value: unknown): string | undefined;
22
+ /**
23
+ * Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
24
+ * an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
25
+ * would match nothing or everything:
26
+ * - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
27
+ * - a lone surrogate (half of a character, sent as U+FFFD);
28
+ * - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
29
+ * hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
30
+ * nothing, or something else (Solr splits a word at a zero-width space).
31
+ * The reason quotes the value with every invisible character escaped (`quoteValue`).
32
+ */
33
+ export declare function textProblem(value: string): string | undefined;
34
+ /**
35
+ * A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
36
+ * with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
37
+ * "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
38
+ * composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
39
+ * query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
40
+ * NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
41
+ * meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
42
+ * forms) that a phrase or a wildcard may mean literally.
43
+ */
44
+ export declare function normalizeText(value: string): string;
45
+ /**
46
+ * Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
47
+ * field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
48
+ * with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
49
+ * only invisible characters or combining marks), no lone surrogate, and no control or format
50
+ * character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
51
+ * character in an id or a field name only ever makes it match nothing. The reason quotes the
52
+ * value visibly.
53
+ */
54
+ export declare function tokenProblem(value: string): string | undefined;
55
+ /** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
56
+ export declare function normalizeToken(value: string): string;
57
+ /**
58
+ * Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
59
+ * (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
60
+ * no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
61
+ * hold a `/`; a value with one is a pasted URL or path, not an id.
62
+ */
63
+ export declare function idProblem(value: string): string | undefined;
64
+ /**
65
+ * Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
66
+ * {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
67
+ * `extras_registerobject_type`). A field name with a space or another character names no field,
68
+ * and CKAN answers with an empty facet and exit 0.
69
+ */
70
+ export declare function fieldProblem(value: string): string | undefined;
71
+ /**
72
+ * Why `value` cannot be sent as a sort expression, or undefined when it can: the
73
+ * {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
74
+ * and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
75
+ * refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
76
+ * not caught: CKAN sorts by its default then, as documented.
77
+ */
78
+ export declare function sortProblem(value: string): string | undefined;
79
+ /**
80
+ * Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
81
+ * item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
82
+ * default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
83
+ */
84
+ export declare function listProblem(items: readonly string[], itemProblem?: Problem<string>): string | undefined;
22
85
  /**
23
86
  * A value the query-string builder can send: a string, a finite number, a boolean
24
- * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
25
- * function or `NaN` would go out as `[object Object]` or `NaN`.
87
+ * or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
88
+ * object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
89
+ * least one value and no `null`/`undefined` entry: the query builder drops those, so
90
+ * `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
91
+ * list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
26
92
  */
27
93
  export declare function queryValueProblem(value: unknown): string | undefined;
28
94
  /**
@@ -2,7 +2,7 @@
2
2
  // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
3
  // enforce them with `assertValid` before any request; the CLI's value parsers call
4
4
  // the same functions, so a rule is written once and the CLI and the library agree.
5
- import { CkanValidationError, cutForMessage } from "./errors.js";
5
+ import { CkanValidationError, cutForMessage, quoteValue } from "./errors.js";
6
6
  /**
7
7
  * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
8
  * something wrong with `value`; otherwise return `value` unchanged. A method that
@@ -15,9 +15,17 @@ export function assertValid(name, value, problem) {
15
15
  throw new CkanValidationError(`Invalid ${name}: ${reason}`);
16
16
  return value;
17
17
  }
18
- /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
18
+ /** Characters that never make a value non-blank: whitespace, controls, format characters, combining marks. */
19
+ const INVISIBLE = /[\s\p{Cc}\p{Cf}\p{M}]/gu;
20
+ /** A lone surrogate: half of a character, which the query string would send as U+FFFD. */
21
+ const LONE_SURROGATE = /[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/;
22
+ /**
23
+ * True when nothing in `value` would match anything: it holds only whitespace, control and
24
+ * format characters (a zero-width space, a soft hyphen) and combining marks. CKAN reads an
25
+ * empty parameter as "not given", and such a value matches nothing, so it is refused as blank.
26
+ */
19
27
  export function isBlank(value) {
20
- return value.trim() === "";
28
+ return value.replace(INVISIBLE, "") === "";
21
29
  }
22
30
  /**
23
31
  * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
@@ -30,35 +38,191 @@ export function blankProblem(value) {
30
38
  return "Expected a non-empty value.";
31
39
  return undefined;
32
40
  }
33
- /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
41
+ /** `U+200B` for the first character of `text` that `pattern` matches. */
42
+ function codePointOf(text, pattern) {
43
+ const ch = pattern.exec(text)?.[0] ?? "";
44
+ return `U+${(ch.codePointAt(0) ?? 0).toString(16).toUpperCase().padStart(4, "0")}`;
45
+ }
46
+ /** A control character inside a text value: C0 and C1, but not TAB, LF or CR (whitespace in a pasted phrase). */
47
+ const TEXT_CONTROL = /[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f-\u009f]/;
48
+ /** A format character inside a text value, but not ZWNJ/ZWJ (U+200C/U+200D), which some scripts and emoji need. */
49
+ const TEXT_FORMAT = /(?![\u200c\u200d])\p{Cf}/u;
50
+ /**
51
+ * Why `value` cannot be sent as a text value (the search text `q`, a tag query, a Solr filter,
52
+ * an `action()` parameter), or undefined when it can. Refused, before any request, as the portal
53
+ * would match nothing or everything:
54
+ * - a value that is blank (`isBlank`): CKAN reads an empty parameter as "no filter";
55
+ * - a lone surrogate (half of a character, sent as U+FFFD);
56
+ * - a control character (but TAB, LF and CR) or a format character (a zero-width space, a soft
57
+ * hyphen, a bidi control; ZWNJ and ZWJ are allowed): pasted along, it makes the search find
58
+ * nothing, or something else (Solr splits a word at a zero-width space).
59
+ * The reason quotes the value with every invisible character escaped (`quoteValue`).
60
+ */
34
61
  export function textProblem(value) {
35
62
  if (typeof value !== "string")
36
63
  return "Expected a string.";
37
- return blankProblem(value);
64
+ if (isBlank(value))
65
+ return "Expected a non-empty value.";
66
+ if (LONE_SURROGATE.test(value))
67
+ return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
68
+ if (TEXT_CONTROL.test(value))
69
+ return `${quoteValue(value)} contains a control character (${codePointOf(value, TEXT_CONTROL)}).`;
70
+ if (TEXT_FORMAT.test(value))
71
+ return `${quoteValue(value)} contains an invisible character (${codePointOf(value, TEXT_FORMAT)}).`;
72
+ return undefined;
38
73
  }
39
- /** A list of non-blank strings (filter queries, facet fields). */
40
- export function textListProblem(value) {
41
- if (!Array.isArray(value))
42
- return "Expected a list of strings.";
43
- if (value.some((v) => typeof v !== "string"))
74
+ /**
75
+ * A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
76
+ * with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
77
+ * "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
78
+ * composed form matches). Inner whitespace is kept. Used for every text value (`q`, the tag
79
+ * query, Solr filters, `action()` parameter values); tokens go through `normalizeToken`.
80
+ * NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
81
+ * meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
82
+ * forms) that a phrase or a wildcard may mean literally.
83
+ */
84
+ export function normalizeText(value) {
85
+ return value.trim().normalize("NFC");
86
+ }
87
+ /** A control or format character in a token (after trimming): every Cc and every Cf. */
88
+ const TOKEN_INVISIBLE = /[\p{Cc}\p{Cf}]/u;
89
+ /**
90
+ * Why `value` cannot be sent as a token, or undefined when it can. A token is an id, a name, a
91
+ * field name or a sort expression: a value CKAN compares exactly. The rules apply to the value
92
+ * with surrounding whitespace trimmed, as it is sent ({@link normalizeToken}): not blank (and not
93
+ * only invisible characters or combining marks), no lone surrogate, and no control or format
94
+ * character at all (a zero-width space, ZWJ, a soft hyphen, a bidi control), since an invisible
95
+ * character in an id or a field name only ever makes it match nothing. The reason quotes the
96
+ * value visibly.
97
+ */
98
+ export function tokenProblem(value) {
99
+ if (typeof value !== "string")
100
+ return "Expected a string.";
101
+ if (isBlank(value))
102
+ return "Expected a non-empty value.";
103
+ const token = value.trim();
104
+ if (LONE_SURROGATE.test(token))
105
+ return `${quoteValue(value)} contains half of a character (a lone surrogate).`;
106
+ if (TOKEN_INVISIBLE.test(token)) {
107
+ return `${quoteValue(value)} contains an invisible or control character (${codePointOf(token, TOKEN_INVISIBLE)}).`;
108
+ }
109
+ return undefined;
110
+ }
111
+ /** A token as it is sent: trimmed and in composed Unicode (NFC). Check it with {@link tokenProblem} first. */
112
+ export function normalizeToken(value) {
113
+ return value.trim().normalize("NFC");
114
+ }
115
+ /**
116
+ * Why `value` cannot be sent as the id or name of a dataset, resource, organization or group
117
+ * (`*_show`), or undefined when it can: the {@link tokenProblem} rules, and on the trimmed value
118
+ * no `/` and not `.` or `..`. CKAN ids are UUIDs and names slugs (`[a-z0-9_-]`), so neither can
119
+ * hold a `/`; a value with one is a pasted URL or path, not an id.
120
+ */
121
+ export function idProblem(value) {
122
+ const problem = tokenProblem(value);
123
+ if (problem !== undefined)
124
+ return problem;
125
+ const id = value.trim();
126
+ if (id.includes("/"))
127
+ return `${quoteValue(value)} is not an id or name: it contains "/".`;
128
+ if (id === "." || id === "..")
129
+ return '"." and ".." are not ids.';
130
+ return undefined;
131
+ }
132
+ /**
133
+ * Why `value` cannot be sent as a Solr field name (a facet field), or undefined when it can: the
134
+ * {@link idProblem} rules, then ASCII letters, digits, `_`, `.` and `-` only (`res_format`,
135
+ * `extras_registerobject_type`). A field name with a space or another character names no field,
136
+ * and CKAN answers with an empty facet and exit 0.
137
+ */
138
+ export function fieldProblem(value) {
139
+ const problem = idProblem(value);
140
+ if (problem !== undefined)
141
+ return problem;
142
+ if (!/^[\w.-]+$/.test(value.trim()))
143
+ return `${quoteValue(value)} is not a Solr field name (letters, digits, _ . -), e.g. res_format.`;
144
+ return undefined;
145
+ }
146
+ /**
147
+ * Why `value` cannot be sent as a sort expression, or undefined when it can: the
148
+ * {@link tokenProblem} rules, then Solr's syntax, a comma-separated list of a field (or function)
149
+ * and a direction, `metadata_modified desc, name asc`. A criterion without `asc`/`desc` is
150
+ * refused here rather than as CKAN's HTTP 400 (`Invalid "sort" parameter`). An unknown field is
151
+ * not caught: CKAN sorts by its default then, as documented.
152
+ */
153
+ export function sortProblem(value) {
154
+ const problem = tokenProblem(value);
155
+ if (problem !== undefined)
156
+ return problem;
157
+ // Split at the commas outside parentheses: a function sort (`div(a,b) desc`) holds commas.
158
+ const criteria = [];
159
+ let depth = 0;
160
+ let from = 0;
161
+ const sort = value.trim();
162
+ for (let i = 0; i < sort.length; i++) {
163
+ const c = sort[i];
164
+ if (c === "(")
165
+ depth++;
166
+ else if (c === ")")
167
+ depth--;
168
+ else if (c === "," && depth === 0) {
169
+ criteria.push(sort.slice(from, i));
170
+ from = i + 1;
171
+ }
172
+ }
173
+ criteria.push(sort.slice(from));
174
+ if (criteria.some((criterion) => !/^\s*\S.*\s(asc|desc)\s*$/i.test(criterion))) {
175
+ return `${quoteValue(value)} is not a sort expression: expected "<field> asc|desc[, <field> asc|desc]", e.g. "metadata_modified desc".`;
176
+ }
177
+ return undefined;
178
+ }
179
+ /**
180
+ * Why `items` cannot be sent as a list, or undefined when it can: not an array of strings, no
181
+ * item at all (an empty list reads as "no filter"), or an item that `itemProblem` refuses (by
182
+ * default {@link tokenProblem}: a blank item is refused, never dropped). The reason names the item.
183
+ */
184
+ export function listProblem(items, itemProblem = tokenProblem) {
185
+ if (!Array.isArray(items))
44
186
  return "Expected a list of strings.";
45
- return blankProblem(value);
187
+ if (items.length === 0)
188
+ return "Expected at least one value.";
189
+ for (let i = 0; i < items.length; i++) {
190
+ const problem = itemProblem(items[i]);
191
+ if (problem !== undefined)
192
+ return items.length === 1 ? problem : `item ${i + 1} of ${items.length}: ${problem}`;
193
+ }
194
+ return undefined;
46
195
  }
47
196
  /**
48
197
  * A value the query-string builder can send: a string, a finite number, a boolean
49
- * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
50
- * function or `NaN` would go out as `[object Object]` or `NaN`.
198
+ * or a Date, or a non-empty list of them (`null`/`undefined` alone mean "not given"). An
199
+ * object, a function or `NaN` would go out as `[object Object]` or `NaN`. A list must hold at
200
+ * least one value and no `null`/`undefined` entry: the query builder drops those, so
201
+ * `{ q: [null] }` ran the action with no `q` at all, over the whole catalogue. A string (or a
202
+ * list entry) must pass {@link textProblem}: it is sent as text (`normalizeText`).
51
203
  */
52
204
  export function queryValueProblem(value) {
205
+ if (Array.isArray(value)) {
206
+ if (value.length === 0)
207
+ return "Expected at least one value; leave the parameter out instead.";
208
+ const missing = value.findIndex((v) => v === null || v === undefined);
209
+ if (missing >= 0) {
210
+ return `item ${missing + 1} of ${value.length} is ${String(value[missing])}; a list holds values only (leave the parameter out instead).`;
211
+ }
212
+ }
53
213
  const ok = (v) => v === null ||
54
214
  v === undefined ||
55
215
  typeof v === "string" ||
56
216
  typeof v === "boolean" ||
57
217
  (typeof v === "number" && Number.isFinite(v)) ||
58
218
  (v instanceof Date && !Number.isNaN(v.getTime()));
59
- if (Array.isArray(value) ? value.every(ok) : ok(value))
60
- return blankProblem(value);
61
- return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
219
+ if (!(Array.isArray(value) ? value.every(ok) : ok(value))) {
220
+ return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
221
+ }
222
+ // A string goes out as text (normalizeText): the text rules apply to it and to each list entry.
223
+ const strings = (Array.isArray(value) ? value : [value]).filter((v) => typeof v === "string");
224
+ const problem = listProblem(strings, textProblem);
225
+ return strings.length === 0 ? undefined : problem;
62
226
  }
63
227
  /**
64
228
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
@@ -1,8 +1,9 @@
1
1
  // Commands over the catalogue: datasets, their resources, publishers, themes
2
2
  // and tags.
3
3
  import { InvalidArgumentError } from "commander";
4
- import { facetLimitProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
5
- import { action, collectNonEmpty, once, parseBoundedInt, parseIntArg, parseNonEmpty, parseOptionValue, renderJson, } from "../shared.js";
4
+ import { facetLimitProblem, fieldProblem, sortProblem, textProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
5
+ import { action, collectValues, once, parseBoundedInt, parseId, parseIntArg, parseOptionValue, parseText, } from "../shared.js";
6
+ import { renderJson } from "../output.js";
6
7
  /**
7
8
  * commander value-parser for a list --limit: 1 or more. CKAN reads `limit=0` as
8
9
  * "no limit" and would send the whole list; leave --limit out for that.
@@ -23,12 +24,12 @@ export function registerCatalogueCommands(program, deps) {
23
24
  program
24
25
  .command("search")
25
26
  .description("Search datasets (Solr query syntax)")
26
- .argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseNonEmpty)
27
+ .argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseText)
27
28
  .option("--rows <n>", "max results (servers cap this, usually at 1000)", once(parseIntArg))
28
29
  .option("--start <n>", "offset for paging", once(parseIntArg))
29
- .option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue))
30
- .option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectNonEmpty)
31
- .option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectNonEmpty)
30
+ .option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue(sortProblem)))
31
+ .option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectValues(textProblem))
32
+ .option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectValues(fieldProblem))
32
33
  .option("--facet-limit <n>", "max values per facet (default 50; -1 = all)", once(parseFacetLimit))
33
34
  .action(action(deps, async ({ client, global, opts }, [query]) => {
34
35
  renderJson(deps, global, await client.packageSearch({
@@ -44,28 +45,28 @@ export function registerCatalogueCommands(program, deps) {
44
45
  program
45
46
  .command("package")
46
47
  .description("Show one dataset by id or name")
47
- .argument("<id>", "id or name", parseNonEmpty)
48
+ .argument("<id>", "id or name", parseId)
48
49
  .action(action(deps, async ({ client, global }, [id]) => {
49
50
  renderJson(deps, global, await client.packageShow(id));
50
51
  }));
51
52
  program
52
53
  .command("resource")
53
54
  .description("Show one resource (distribution) by id")
54
- .argument("<id>", "resource id", parseNonEmpty)
55
+ .argument("<id>", "resource id", parseId)
55
56
  .action(action(deps, async ({ client, global }, [id]) => {
56
57
  renderJson(deps, global, await client.resourceShow(id));
57
58
  }));
58
59
  program
59
60
  .command("organization")
60
61
  .description("Show one organization (publisher) by id or name")
61
- .argument("<id>", "id or name", parseNonEmpty)
62
+ .argument("<id>", "id or name", parseId)
62
63
  .action(action(deps, async ({ client, global }, [id]) => {
63
64
  renderJson(deps, global, await client.organizationShow(id));
64
65
  }));
65
66
  program
66
67
  .command("group")
67
68
  .description("Show one group (theme/category) by id or name")
68
- .argument("<id>", "id or name", parseNonEmpty)
69
+ .argument("<id>", "id or name", parseId)
69
70
  .action(action(deps, async ({ client, global }, [id]) => {
70
71
  renderJson(deps, global, await client.groupShow(id));
71
72
  }));
@@ -109,7 +110,7 @@ export function registerCatalogueCommands(program, deps) {
109
110
  program
110
111
  .command("tags")
111
112
  .description("List tags")
112
- .option("--query <substring>", "only tags containing this substring", once(parseOptionValue))
113
+ .option("--query <substring>", "only tags containing this substring", once(parseOptionValue(textProblem)))
113
114
  .action(action(deps, async ({ client, global, opts }) => {
114
115
  renderJson(deps, global, await client.tagList({ query: opts["query"] }));
115
116
  }));
@@ -1,40 +1,48 @@
1
1
  // Commands about the portal itself rather than its datasets.
2
2
  import { InvalidArgumentError } from "commander";
3
- import { action, renderJson, toEngineOptions } from "../shared.js";
3
+ import { action, toEngineOptions } from "../shared.js";
4
+ import { renderJson } from "../output.js";
4
5
  import { checkPortals } from "@maschinenlesbar.org/ckan-client/internal/portals";
5
6
  import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
6
- import { isBlank } from "@maschinenlesbar.org/ckan-client/internal/validate";
7
- import { cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
7
+ import { normalizeToken, textProblem, tokenProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
8
+ import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
8
9
  /**
9
- * A typed value as a message quotes it: cut at MAX_MESSAGE_VALUE_LENGTH, after its
10
- * credentials are redacted (a cut could otherwise leave part of a password without the
11
- * "@" the argv redaction keys on).
10
+ * A typed value as a message quotes it: visibly (`quoteValue`: invisible and control characters
11
+ * escaped, cut at MAX_MESSAGE_VALUE_LENGTH), after its credentials are redacted (a cut could
12
+ * otherwise leave part of a password without the "@" the argv redaction keys on).
12
13
  */
13
14
  function shown(value) {
14
- return cutForMessage(redactUrl(value));
15
+ return quoteValue(redactUrl(value));
15
16
  }
16
- /** commander accumulator for repeatable `key=value` pairs into a record. */
17
+ /**
18
+ * commander accumulator for repeatable `key=value` pairs into a record. The library's rules,
19
+ * as a usage error: the name is a token (`tokenProblem`; sent trimmed, so ` q` and `q` are the
20
+ * same name, and giving it twice is refused), the value is text (`textProblem`: not blank, no
21
+ * control or invisible character).
22
+ */
17
23
  function collectKeyValue(value, previous = {}) {
18
24
  const eq = value.indexOf("=");
19
25
  // Throw commander's InvalidArgumentError (not a CkanError) so the failure is
20
26
  // formatted as a usage error with `error:` prefix and help, consistent with
21
27
  // other parse-time flag validation (e.g. `--rows abc`).
22
- const key = value.slice(0, Math.max(eq, 0));
28
+ if (eq < 0)
29
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}. Expected key=value.`);
30
+ const key = value.slice(0, eq);
23
31
  // A blank key (`=x`, ` =x`) names no parameter: the library refuses it too.
24
- if (eq < 0 || isBlank(key)) {
25
- throw new InvalidArgumentError(`Invalid --param "${shown(value)}". Expected key=value.`);
26
- }
27
- // Reject a duplicated key rather than silently overwriting the earlier value.
28
- if (Object.prototype.hasOwnProperty.call(previous, key)) {
29
- throw new InvalidArgumentError(`Duplicate --param key "${shown(key)}".`);
32
+ const keyProblem = tokenProblem(key);
33
+ if (keyProblem !== undefined)
34
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the name: ${keyProblem}`);
35
+ // Reject a duplicated key (as it is sent, trimmed) rather than silently overwriting the earlier value.
36
+ const name = normalizeToken(key);
37
+ if (Object.keys(previous).some((k) => normalizeToken(k) === name)) {
38
+ throw new InvalidArgumentError(`Duplicate --param key ${shown(name)}.`);
30
39
  }
31
40
  // A blank value is refused by the library as well (CKAN reads an empty
32
- // parameter as unset, so `--param q=` would run the action unfiltered); here it
33
- // is a usage error that names the flag.
41
+ // parameter as unset, so `--param q=` would run the action unfiltered).
34
42
  const paramValue = value.slice(eq + 1);
35
- if (isBlank(paramValue)) {
36
- throw new InvalidArgumentError(`Invalid --param "${shown(value)}". The value must not be blank.`);
37
- }
43
+ const valueProblem = textProblem(paramValue);
44
+ if (valueProblem !== undefined)
45
+ throw new InvalidArgumentError(`Invalid --param ${shown(value)}: the value: ${valueProblem}`);
38
46
  // A computed key in a literal is an own data property, even for `__proto__`.
39
47
  return { ...previous, [key]: paramValue };
40
48
  }
@@ -58,4 +58,20 @@ export interface StdoutBacklog {
58
58
  * process alive until then, so a held record is never lost at exit.
59
59
  */
60
60
  export declare function stderrAfterStdout(stdout: StdoutBacklog, write: (text: string) => void): (text: string) => void;
61
+ /** What `stdoutWriter` needs of stdout. */
62
+ interface Stdout {
63
+ write(text: string): unknown;
64
+ emit(event: "error", err: Error): unknown;
65
+ readonly fd?: number;
66
+ }
67
+ /**
68
+ * A writer for stdout that never drops a byte silently. When stdout is a regular file
69
+ * (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
70
+ * file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
71
+ * the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
72
+ * `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
73
+ * exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
74
+ */
75
+ export declare function stdoutWriter(stdout?: Stdout, write?: (fd: number, data: Buffer) => void): (text: string) => void;
61
76
  export declare const defaultIO: CliIO;
77
+ export {};
@@ -1,6 +1,7 @@
1
1
  // I/O seam for the CLI. Everything the CLI writes goes through a CliIO object so
2
2
  // tests can capture output instead of hitting the real stdout/stderr, and the
3
3
  // environment is injected so tests never depend on the real process.env.
4
+ import { fstatSync, writeSync } from "node:fs";
4
5
  import { createLogger } from "./log.js";
5
6
  /** The deps' logger, or one that writes text records through `io.err`. */
6
7
  export function logOf(deps) {
@@ -78,8 +79,61 @@ export function stderrAfterStdout(stdout, write) {
78
79
  };
79
80
  }
80
81
  const stderrLine = stderrAfterStdout(process.stdout, (text) => process.stderr.write(text + "\n"));
82
+ /** A short pause for a non-blocking descriptor that is full (EAGAIN), without a busy loop. */
83
+ const pause = new Int32Array(new SharedArrayBuffer(4));
84
+ /**
85
+ * Write all of `data` to `fd`, or throw. `writeSync` may write less than asked (a file-size
86
+ * limit, a full disk): the rest is written in a loop, and the call that cannot write throws
87
+ * (EFBIG, ENOSPC). A descriptor the shell made non-blocking (a pipe through `/dev/fd/N`) answers
88
+ * EAGAIN while the reader is behind: wait a moment and go on.
89
+ */
90
+ function writeAll(fd, data) {
91
+ for (let at = 0; at < data.length;) {
92
+ try {
93
+ at += writeSync(fd, data, at);
94
+ }
95
+ catch (err) {
96
+ if (err.code !== "EAGAIN")
97
+ throw err;
98
+ Atomics.wait(pause, 0, 0, 5);
99
+ }
100
+ }
101
+ }
102
+ /**
103
+ * A writer for stdout that never drops a byte silently. When stdout is a regular file
104
+ * (`> out.json`), Node's stream wrote it with one `writeSync` and ignored a short count, so a
105
+ * file-size limit or a full disk cut the output short with exit 0 and nothing on stderr. Here
106
+ * the text is written with `writeAll`, and a failure (EFBIG, ENOSPC) is emitted as stdout's
107
+ * `error`, which `handleOutputErrors` turns into an ERROR record of `ckan.output` and
108
+ * exit 1. A pipe, a socket or a terminal goes through the stream as before (EPIPE stays quiet).
109
+ */
110
+ export function stdoutWriter(stdout = process.stdout, write = writeAll) {
111
+ let toFile;
112
+ return (text) => {
113
+ toFile ??= stdout.fd !== undefined && isRegularFile(stdout.fd);
114
+ if (!toFile) {
115
+ stdout.write(text);
116
+ return;
117
+ }
118
+ try {
119
+ write(stdout.fd, Buffer.from(text, "utf8"));
120
+ }
121
+ catch (err) {
122
+ stdout.emit("error", err instanceof Error ? err : new Error(String(err)));
123
+ }
124
+ };
125
+ }
126
+ function isRegularFile(fd) {
127
+ try {
128
+ return fstatSync(fd).isFile();
129
+ }
130
+ catch {
131
+ return false;
132
+ }
133
+ }
134
+ const stdoutText = stdoutWriter();
81
135
  export const defaultIO = {
82
- out: (text) => process.stdout.write(text + "\n"),
136
+ out: (text) => stdoutText(text + "\n"),
83
137
  // A record never lands inside the data when both streams share a pipe.
84
138
  err: stderrLine,
85
139
  };
@@ -0,0 +1,26 @@
1
+ import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
2
+ import type { CliDeps } from "./io.js";
3
+ import type { GlobalOptions } from "./shared.js";
4
+ /**
5
+ * Printing the answer failed: an answer too deeply nested or too large to print. An ERROR
6
+ * record of `ckan.output`, exit 1.
7
+ */
8
+ export declare class OutputError extends CkanError {
9
+ }
10
+ /**
11
+ * Escape the characters JSON.stringify leaves raw although a terminal acts on them. It
12
+ * escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is the 8-bit
13
+ * form of CSI) or the bidi formatting characters (isBidiControl), which reorder the text that
14
+ * follows. The output is server data, so escape them; the result is equivalent, valid JSON
15
+ * (these characters only occur inside strings). Checked by char code so the source stays
16
+ * free of control bytes.
17
+ *
18
+ * Each such character becomes six, so the output is measured first: one longer than V8's
19
+ * longest string is an OutputError ("too large to print"), before anything is built. Then the
20
+ * output is built in chunks of ESCAPE_CHUNK characters, each made flat at once, and joined
21
+ * once: one `+=` per character made a rope of 100 M pieces on an answer of DEL characters, and
22
+ * the process died of a heap out-of-memory crash.
23
+ */
24
+ export declare function escapeControlChars(json: string): string;
25
+ /** Render a JSON value to stdout, pretty by default, compact with --compact. */
26
+ export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;