@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/README.md +21 -7
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
  15. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
  16. package/dist/src/cli/commands/catalogue.js +12 -11
  17. package/dist/src/cli/commands/portal.js +28 -20
  18. package/dist/src/cli/io.d.ts +16 -0
  19. package/dist/src/cli/io.js +55 -1
  20. package/dist/src/cli/output.d.ts +26 -0
  21. package/dist/src/cli/output.js +151 -0
  22. package/dist/src/cli/run.d.ts +0 -7
  23. package/dist/src/cli/run.js +80 -17
  24. package/dist/src/cli/shared.d.ts +14 -25
  25. package/dist/src/cli/shared.js +40 -102
  26. package/package.json +1 -1
@@ -0,0 +1,26 @@
1
+ import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
2
+ import type { CliDeps } from "./io.js";
3
+ import type { GlobalOptions } from "./shared.js";
4
+ /**
5
+ * Printing the answer failed: an answer too deeply nested or too large to print. An ERROR
6
+ * record of `ckan.output`, exit 1.
7
+ */
8
+ export declare class OutputError extends CkanError {
9
+ }
10
+ /**
11
+ * Escape the characters JSON.stringify leaves raw although a terminal acts on them. It
12
+ * escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is the 8-bit
13
+ * form of CSI) or the bidi formatting characters (isBidiControl), which reorder the text that
14
+ * follows. The output is server data, so escape them; the result is equivalent, valid JSON
15
+ * (these characters only occur inside strings). Checked by char code so the source stays
16
+ * free of control bytes.
17
+ *
18
+ * Each such character becomes six, so the output is measured first: one longer than V8's
19
+ * longest string is an OutputError ("too large to print"), before anything is built. Then the
20
+ * output is built in chunks of ESCAPE_CHUNK characters, each made flat at once, and joined
21
+ * once: one `+=` per character made a rope of 100 M pieces on an answer of DEL characters, and
22
+ * the process died of a heap out-of-memory crash.
23
+ */
24
+ export declare function escapeControlChars(json: string): string;
25
+ /** Render a JSON value to stdout, pretty by default, compact with --compact. */
26
+ export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
@@ -0,0 +1,151 @@
1
+ // The output sink: everything the CLI writes as data, to stdout. One place for the rules the
2
+ // maschinenlesbar.org CLIs learned one bug at a time: terminal controls escaped, a deep or huge
3
+ // answer printed (or refused) cleanly, as an ERROR record of `ckan.output`.
4
+ import { constants } from "node:buffer";
5
+ import { isBidiControl } from "@maschinenlesbar.org/ckan-client/internal/engine";
6
+ import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
7
+ /**
8
+ * Printing the answer failed: an answer too deeply nested or too large to print. An ERROR
9
+ * record of `ckan.output`, exit 1.
10
+ */
11
+ export class OutputError extends CkanError {
12
+ }
13
+ /** The longest string V8 makes (2^29 - 24 characters on 64-bit): a longer JSON text can't be printed at once. */
14
+ const MAX_TEXT_LENGTH = constants.MAX_STRING_LENGTH;
15
+ /**
16
+ * JSON.stringify, pretty or compact. Printing can fail two ways, both an OutputError (exit 1)
17
+ * that names the cause, never "Unexpected error":
18
+ * - an answer nested too deeply (hostile or broken) overflows the stack: "nested too deeply";
19
+ * - a text longer than V8's longest string ("Invalid string length"): "too large to print".
20
+ * "try --compact" is added only when the compact form of this very answer prints: it is probed,
21
+ * not assumed, because whether compact output survives deep nesting depends on the Node version
22
+ * (on Node 26 only the fast path is iterative) and on the machine's stack.
23
+ *
24
+ * Compact output takes V8's fast path (no gap) unless the answer holds a huge string: on Node 26
25
+ * the fast path aborts the process ("Zone Allocation failed", exit 134, a native dump on stderr)
26
+ * on a string of some 250 MB that JSON.parse made, while the gap path (an empty gap, the same
27
+ * text) prints it. The fast path stays the default because on Node 26 only it is iterative and
28
+ * prints nesting the gap path refuses (~7,000 levels).
29
+ */
30
+ function stringifyJson(value, compact) {
31
+ try {
32
+ return compact ? stringifyCompact(value) : JSON.stringify(value, null, 2);
33
+ }
34
+ catch (err) {
35
+ if (!(err instanceof RangeError))
36
+ throw err;
37
+ const deep = /call stack/i.test(err.message);
38
+ const cause = deep ? "nested too deeply" : `too large (more than ${MAX_TEXT_LENGTH} characters of JSON)`;
39
+ const hint = !compact && compactPrints(value) ? "; try --compact" : "";
40
+ throw new OutputError(`The answer is ${cause} to ${compact ? "print" : "pretty-print"}${hint}.`, { cause: err });
41
+ }
42
+ }
43
+ /** Compact JSON: the fast path, or the gap path for an answer holding a huge string. */
44
+ function stringifyCompact(value) {
45
+ return holdsHugeString(value) ? JSON.stringify(value, null, "") : JSON.stringify(value);
46
+ }
47
+ /** Whether `--compact` prints this answer on this runtime (the hint after a failed pretty-print). */
48
+ function compactPrints(value) {
49
+ try {
50
+ return stringifyCompact(value).length <= MAX_TEXT_LENGTH;
51
+ }
52
+ catch {
53
+ return false;
54
+ }
55
+ }
56
+ /** A string or key this long sends --compact through the gap path: a quarter of where Node 26's fast path aborts. */
57
+ const HUGE_STRING_LENGTH = 64 * 1024 * 1024;
58
+ /**
59
+ * Whether `value` holds a string or key of HUGE_STRING_LENGTH or more. Iterative, so nesting
60
+ * costs no stack, linear, and without an allocation per value: an array is walked by index, an
61
+ * object by `for…in` (own keys only: an answer's objects have none inherited), and only nested
62
+ * objects and arrays are kept for later. (`Object.entries` made a pair per element: on an array
63
+ * of 48M numbers that took 7 GB and crashed.)
64
+ */
65
+ function holdsHugeString(value) {
66
+ if (typeof value === "string")
67
+ return value.length >= HUGE_STRING_LENGTH;
68
+ const pending = [];
69
+ if (typeof value === "object" && value !== null)
70
+ pending.push(value);
71
+ for (let item = pending.pop(); item !== undefined; item = pending.pop()) {
72
+ if (Array.isArray(item)) {
73
+ for (let i = 0; i < item.length; i++) {
74
+ const element = item[i];
75
+ if (typeof element === "string") {
76
+ if (element.length >= HUGE_STRING_LENGTH)
77
+ return true;
78
+ }
79
+ else if (typeof element === "object" && element !== null)
80
+ pending.push(element);
81
+ }
82
+ continue;
83
+ }
84
+ const record = item;
85
+ for (const key in record) {
86
+ if (key.length >= HUGE_STRING_LENGTH)
87
+ return true;
88
+ const element = record[key];
89
+ if (typeof element === "string") {
90
+ if (element.length >= HUGE_STRING_LENGTH)
91
+ return true;
92
+ }
93
+ else if (typeof element === "object" && element !== null)
94
+ pending.push(element);
95
+ }
96
+ }
97
+ return false;
98
+ }
99
+ /** True for a UTF-16 unit `escapeControlChars` writes as `\uXXXX`. */
100
+ function escaped(c) {
101
+ return (c >= 0x7f && c <= 0x9f) || isBidiControl(c);
102
+ }
103
+ /** How many input characters `escapeControlChars` turns into one piece of output at a time. */
104
+ const ESCAPE_CHUNK = 1 << 16;
105
+ /**
106
+ * Escape the characters JSON.stringify leaves raw although a terminal acts on them. It
107
+ * escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is the 8-bit
108
+ * form of CSI) or the bidi formatting characters (isBidiControl), which reorder the text that
109
+ * follows. The output is server data, so escape them; the result is equivalent, valid JSON
110
+ * (these characters only occur inside strings). Checked by char code so the source stays
111
+ * free of control bytes.
112
+ *
113
+ * Each such character becomes six, so the output is measured first: one longer than V8's
114
+ * longest string is an OutputError ("too large to print"), before anything is built. Then the
115
+ * output is built in chunks of ESCAPE_CHUNK characters, each made flat at once, and joined
116
+ * once: one `+=` per character made a rope of 100 M pieces on an answer of DEL characters, and
117
+ * the process died of a heap out-of-memory crash.
118
+ */
119
+ export function escapeControlChars(json) {
120
+ let count = 0;
121
+ for (let i = 0; i < json.length; i++)
122
+ if (escaped(json.charCodeAt(i)))
123
+ count++;
124
+ if (count === 0)
125
+ return json;
126
+ const length = json.length + 5 * count;
127
+ if (length > MAX_TEXT_LENGTH) {
128
+ throw new OutputError(`The answer is too large to print (${length} characters once its control characters are escaped).`);
129
+ }
130
+ const chunks = [];
131
+ for (let start = 0; start < json.length; start += ESCAPE_CHUNK) {
132
+ const end = Math.min(start + ESCAPE_CHUNK, json.length);
133
+ const parts = [];
134
+ let from = start;
135
+ for (let i = start; i < end; i++) {
136
+ const c = json.charCodeAt(i);
137
+ if (!escaped(c))
138
+ continue;
139
+ parts.push(json.slice(from, i), "\\u" + c.toString(16).padStart(4, "0"));
140
+ from = i + 1;
141
+ }
142
+ parts.push(json.slice(from, end));
143
+ chunks.push(parts.join(""));
144
+ }
145
+ return chunks.join("");
146
+ }
147
+ /** Render a JSON value to stdout, pretty by default, compact with --compact. */
148
+ export function renderJson(deps, global, value) {
149
+ const text = escapeControlChars(stringifyJson(value, global.compact === true));
150
+ deps.io.out(text);
151
+ }
@@ -1,13 +1,6 @@
1
1
  import { type Command } from "commander";
2
2
  import { type CliDeps } from "./io.js";
3
3
  import { type Logger } from "./log.js";
4
- /**
5
- * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
6
- * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
7
- * covers a URL that does not parse; a fallback behind the exact strings of
8
- * `withRedactedOutput`.
9
- */
10
- export declare function redactUserinfo(text: string): string;
11
4
  /** The secrets of a run, and the two ways they are replaced. */
12
5
  export interface Redaction {
13
6
  /**
@@ -4,8 +4,9 @@
4
4
  import { CommanderError } from "commander";
5
5
  import { buildProgram, defaultDeps } from "./program.js";
6
6
  import { logOf } from "./io.js";
7
+ import { OutputError } from "./output.js";
7
8
  import { createLogger, logFormatFromArgv } from "./log.js";
8
- import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "@maschinenlesbar.org/ckan-client/internal/errors";
9
+ import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, redactUrlUserinfo, typedUrlCredentials, } from "@maschinenlesbar.org/ckan-client/internal/errors";
9
10
  /** The exit code of a usage error: commander's own for a rejected option value. */
10
11
  const USAGE_EXIT = 1;
11
12
  /**
@@ -64,7 +65,7 @@ function writeCommanderErr(command, deps, state, str) {
64
65
  return;
65
66
  if (text.startsWith("error: ")) {
66
67
  state.errorLogged = true;
67
- log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
68
+ log.error("cli", escapeCommanderError(text.slice("error: ".length)).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
68
69
  return;
69
70
  }
70
71
  if (!state.errorLogged) {
@@ -76,13 +77,35 @@ function writeCommanderErr(command, deps, state, str) {
76
77
  log.info("cli", line.trimEnd());
77
78
  }
78
79
  /**
79
- * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
80
- * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
81
- * covers a URL that does not parse; a fallback behind the exact strings of
82
- * `withRedactedOutput`.
80
+ * Escape the characters a terminal acts on in one commander error message: CR and LF
81
+ * become `\r` and `\n` (as a log record writes them, `escapeForRecord`), the other C0
82
+ * controls (tab excepted), DEL, C1 and Unicode format characters (bidi overrides,
83
+ * zero-width characters) `\uXXXX`. Commander repeats a rejected argument raw
84
+ * ("command-argument value '<value>' is invalid"), and a value can come from a
85
+ * pipeline: a newline in it could forge a record, an override reorder the line. The
86
+ * record escapes most of these too; this keeps the invisible format characters visible,
87
+ * as the library's own refusals show them (`quoteValue`).
83
88
  */
84
- export function redactUserinfo(text) {
85
- return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
89
+ function escapeTerminalText(text) {
90
+ return text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f]|\p{Cf}/gu, (ch) => ch === "\n"
91
+ ? "\\n"
92
+ : ch === "\r"
93
+ ? "\\r"
94
+ : Array.from({ length: ch.length }, (_, i) => `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`).join(""));
95
+ }
96
+ /**
97
+ * escapeTerminalText for one commander error message, except for the
98
+ * `\n(Did you mean …?)` line commander appends after an unknown command or option.
99
+ * That hint is built from this CLI's own command and option names, and it can only
100
+ * end the message: commander quotes the user's value (`unknown command '<value>'`), so
101
+ * a value that contains the same text is followed by its closing quote and stays
102
+ * escaped.
103
+ */
104
+ function escapeCommanderError(message) {
105
+ const hint = /\n\(Did you mean [^\n]*\?\)$/.exec(message);
106
+ if (hint === null)
107
+ return escapeTerminalText(message);
108
+ return escapeTerminalText(message.slice(0, hint.index)) + hint[0];
86
109
  }
87
110
  /**
88
111
  * The options whose value is a base URL (`--portal` takes a portal's URL as well as its
@@ -103,6 +126,33 @@ function flagValues(argv, flags) {
103
126
  });
104
127
  return found;
105
128
  }
129
+ /** The most URLs `urlsWithin` reads inside one argument, after the one it may start with. */
130
+ const MAX_URLS_WITHIN = 64;
131
+ /** A character a URL scheme may hold after its first letter. */
132
+ const SCHEME_CHAR = /[A-Za-z0-9+.-]/;
133
+ /**
134
+ * `value` and the rest of it from each place a URL starts inside it (`<scheme>://`, the scheme
135
+ * starting with a letter), for `credentialsIn`, which reads a URL at the start of its input only.
136
+ * The library sends a text trimmed, so a padded `" https://user:pw@host"`, one after a dash or
137
+ * inside a phrase (`search "elbe https://…"`, `--fq organization:https://…`) reaches the request
138
+ * URL of an error record, a refusal and commander's echo. Each `://` is found with indexOf and its
139
+ * scheme by walking back, never by a pattern retried at every position (quadratic: 40 000 scheme
140
+ * characters took ~3 s); at most MAX_URLS_WITHIN are read from one argument, since each is parsed
141
+ * whole.
142
+ */
143
+ function urlsWithin(value) {
144
+ const found = [value];
145
+ for (let at = value.indexOf("://"); at !== -1 && found.length <= MAX_URLS_WITHIN; at = value.indexOf("://", at + 3)) {
146
+ let start = at;
147
+ while (start > 0 && SCHEME_CHAR.test(value[start - 1]))
148
+ start--;
149
+ while (start < at && !/[A-Za-z]/.test(value[start]))
150
+ start++;
151
+ if (start > 0 && start < at)
152
+ found.push(value.slice(start));
153
+ }
154
+ return found;
155
+ }
106
156
  /**
107
157
  * The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
108
158
  * its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
@@ -113,20 +163,31 @@ function flagValues(argv, flags) {
113
163
  * `/`; the exact strings can. Without credentials the text passes through unchanged.
114
164
  */
115
165
  export function redactionFor(argv, env) {
116
- // An `--option=value` token is echoed as its value alone.
117
- const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
166
+ // An `--option=value` token is echoed as its value alone; a `key=value` pair (`--param
167
+ // id=https://user:pw@host`) carries its value after the "=", which a URL there starts.
168
+ const values = argv.flatMap((token) => {
169
+ const eq = token.indexOf("=");
170
+ if (eq < 0)
171
+ return [token];
172
+ return token.startsWith("-") ? [token.slice(eq + 1)] : [token, token.slice(eq + 1)];
173
+ });
118
174
  // A base URL typed without its scheme is read as if it had one (anywhere else a bare
119
175
  // `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
120
- const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
121
- const sources = [...values, ...baseUrls];
176
+ // Read as a URL however its scheme is typed: missing, special without "//", or mistyped
177
+ // (`http//u:pw@h`, the colon left out): typedUrlCredentials.
178
+ const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""];
179
+ const sources = [...values.flatMap(urlsWithin).map(credentialsIn), ...baseUrls.map(typedUrlCredentials)];
122
180
  const secrets = new Set();
123
181
  const encoded = new Set();
124
182
  const echoed = new Set();
125
183
  const passwords = new Set();
126
- for (const source of sources) {
127
- for (const secret of credentialsIn(source)) {
184
+ for (const found of sources) {
185
+ for (const secret of found) {
128
186
  secrets.add(secret);
129
187
  secrets.add(JSON.stringify(secret).slice(1, -1));
188
+ // As commander's echo and the library's quoteValue show it: invisible characters escaped.
189
+ secrets.add(escapeTerminalText(secret));
190
+ secrets.add(escapeTerminalText(JSON.stringify(secret).slice(1, -1)));
130
191
  // A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
131
192
  // 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
132
193
  encoded.add(encodeURIComponent(`${secret}@`));
@@ -148,7 +209,7 @@ export function redactionFor(argv, env) {
148
209
  const echoedList = [...echoed].sort((a, b) => b.length - a.length);
149
210
  const passwordList = [...passwords].sort((a, b) => b.length - a.length);
150
211
  const out = (text) => {
151
- let result = redactUserinfo(redactCredentials(text, list));
212
+ let result = redactUrlUserinfo(redactCredentials(text, list));
152
213
  for (const form of encoded)
153
214
  result = result.split(form).join("***%40");
154
215
  return redactSecrets(result, echoedList);
@@ -223,10 +284,12 @@ export function processLogger(argv, env = process.env) {
223
284
  * connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
224
285
  * envelope, the wrong result shape, an unknown charset — the portal's answer as much as
225
286
  * an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
226
- * envelope, `CkanActionError`), else `cli` (a response nested too deeply to print is
227
- * about printing the answer, not its shape).
287
+ * envelope, `CkanActionError`), `output` for an answer too deeply nested or too large to print
288
+ * (`OutputError`: printing it, not its shape), else `cli`.
228
289
  */
229
290
  function areaOf(err) {
291
+ if (err instanceof OutputError)
292
+ return "output";
230
293
  if (err instanceof CkanNetworkError)
231
294
  return "http";
232
295
  if (err instanceof CkanParseError || err instanceof CkanActionError)
@@ -1,12 +1,12 @@
1
1
  import { type CliDeps } from "./io.js";
2
2
  import { type EngineOptions, type RetryEvent } from "@maschinenlesbar.org/ckan-client/internal/engine";
3
+ import { type Problem } from "@maschinenlesbar.org/ckan-client/internal/validate";
3
4
  /** commander value-parser: a non-negative integer. */
4
5
  export declare function parseIntArg(value: string): number;
5
- /**
6
- * commander value-parser: a value that is not blank (the library's blankProblem
7
- * rule, reported as a usage error before the client is built).
8
- */
9
- export declare function parseNonEmpty(value: string): string;
6
+ /** commander value-parser for free text (the search text): the library's `textProblem`. */
7
+ export declare const parseText: (value: string) => string;
8
+ /** commander value-parser for the id or name of a dataset, resource, organization or group: the library's `idProblem`. */
9
+ export declare const parseId: (value: string) => string;
10
10
  /**
11
11
  * commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
12
12
  * the library's headerValueProblem rule (not blank, no control characters but tab,
@@ -14,14 +14,14 @@ export declare function parseNonEmpty(value: string): string;
14
14
  */
15
15
  export declare function parseHeaderValue(value: string): string;
16
16
  /**
17
- * commander value-parser for a text option's value: not blank, and not another
18
- * long option. commander takes the next argument as the value of an option that
19
- * needs one, even when it is a flag, so a forgotten value (`search --fq --rows 5`)
20
- * silently shifted the rest into the query (`fq=--rows`, `q=5`) and still exited
21
- * 0. No CKAN filter, sort, facet field or tag query starts with `--`; a single `-`
22
- * (a negated Solr filter, `-organization:x`) stays allowed.
17
+ * A commander value-parser for an option's value: not another long option, then the library's
18
+ * `problem` rule (`textProblem` for a filter or tag query, `sortProblem`, `fieldProblem`).
19
+ * commander takes the next argument as the value of an option that needs one, even when it is
20
+ * a flag, so a forgotten value (`search --fq --rows 5`) silently shifted the rest into the
21
+ * query (`fq=--rows`, `q=5`) and still exited 0. No CKAN filter, sort, facet field or tag query
22
+ * starts with `--`; a single `-` (a negated Solr filter, `-organization:x`) stays allowed.
23
23
  */
24
- export declare function parseOptionValue(value: string): string;
24
+ export declare function parseOptionValue(problem: Problem<string>): (value: string) => string;
25
25
  /**
26
26
  * Wrap a commander value-parser for a single-valued option so that a second
27
27
  * occurrence is a usage error. commander otherwise keeps only the last value, so
@@ -30,8 +30,8 @@ export declare function parseOptionValue(value: string): string;
30
30
  * commander hands the parser as `previous`); build it per program (`buildProgram`).
31
31
  */
32
32
  export declare function once<T>(parse: (value: string) => T): (value: string) => T;
33
- /** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
34
- export declare function collectNonEmpty(value: string, previous?: string[]): string[];
33
+ /** commander accumulator for a repeatable option (`--fq`, `--facet`), each value checked by parseOptionValue(problem). */
34
+ export declare function collectValues(problem: Problem<string>): (value: string, previous?: string[]) => string[];
35
35
  /** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
36
36
  export declare function parseBoundedInt(min: number, max: number): (value: string) => number;
37
37
  /**
@@ -55,17 +55,6 @@ export interface GlobalOptions {
55
55
  }
56
56
  /** Translate resolved global CLI options into client EngineOptions. */
57
57
  export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
58
- /**
59
- * Escape the characters JSON.stringify leaves raw although a terminal acts on them.
60
- * It escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is
61
- * the 8-bit form of CSI) or the bidi formatting characters (isBidiControl), which
62
- * reorder the text that follows. The output is server data, so escape them; the
63
- * result is equivalent, valid JSON (these characters only occur inside strings).
64
- * Checked by char code so the source stays free of control bytes.
65
- */
66
- export declare function escapeControlChars(json: string): string;
67
- /** Render a JSON value to stdout, pretty by default, compact with --compact. */
68
- export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
69
58
  /**
70
59
  * Log a warning (`ckan.http`) when the effective base URL (--portal >
71
60
  * --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
@@ -1,11 +1,11 @@
1
- // Shared helpers used across CLI command groups: option parsers, the global
2
- // option resolver, and the JSON result renderer.
1
+ // Shared helpers used across CLI command groups: option parsers and the global
2
+ // option resolver. Results are written by output.ts.
3
3
  import { InvalidArgumentError } from "commander";
4
4
  import { logOf } from "./io.js";
5
- import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "@maschinenlesbar.org/ckan-client/internal/engine";
6
- import { CkanError, cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
5
+ import { cleartextProblem, DEFAULT_BASE_URL } from "@maschinenlesbar.org/ckan-client/internal/engine";
6
+ import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
7
7
  import { findPortal } from "@maschinenlesbar.org/ckan-client/internal/portals";
8
- import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
8
+ import { baseUrlProblem, countProblem, headerValueProblem, idProblem, intRangeProblem, textProblem, } from "@maschinenlesbar.org/ckan-client/internal/validate";
9
9
  import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
10
10
  /** commander value-parser: a non-negative integer. */
11
11
  export function parseIntArg(value) {
@@ -24,15 +24,22 @@ export function parseIntArg(value) {
24
24
  return n;
25
25
  }
26
26
  /**
27
- * commander value-parser: a value that is not blank (the library's blankProblem
28
- * rule, reported as a usage error before the client is built).
27
+ * A commander value-parser for the library's `problem` rule (`textProblem`, `idProblem`, …),
28
+ * reported as a usage error before the client is built. The value goes on as typed: the
29
+ * library canonicalises it (`normalizeText`, `normalizeToken`).
29
30
  */
30
- export function parseNonEmpty(value) {
31
- const reason = blankProblem(value);
32
- if (reason !== undefined)
33
- throw new InvalidArgumentError(reason);
34
- return value;
31
+ function parseWith(problem) {
32
+ return (value) => {
33
+ const reason = problem(value);
34
+ if (reason !== undefined)
35
+ throw new InvalidArgumentError(reason);
36
+ return value;
37
+ };
35
38
  }
39
+ /** commander value-parser for free text (the search text): the library's `textProblem`. */
40
+ export const parseText = parseWith(textProblem);
41
+ /** commander value-parser for the id or name of a dataset, resource, organization or group: the library's `idProblem`. */
42
+ export const parseId = parseWith(idProblem);
36
43
  /**
37
44
  * commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
38
45
  * the library's headerValueProblem rule (not blank, no control characters but tab,
@@ -45,18 +52,21 @@ export function parseHeaderValue(value) {
45
52
  return value;
46
53
  }
47
54
  /**
48
- * commander value-parser for a text option's value: not blank, and not another
49
- * long option. commander takes the next argument as the value of an option that
50
- * needs one, even when it is a flag, so a forgotten value (`search --fq --rows 5`)
51
- * silently shifted the rest into the query (`fq=--rows`, `q=5`) and still exited
52
- * 0. No CKAN filter, sort, facet field or tag query starts with `--`; a single `-`
53
- * (a negated Solr filter, `-organization:x`) stays allowed.
55
+ * A commander value-parser for an option's value: not another long option, then the library's
56
+ * `problem` rule (`textProblem` for a filter or tag query, `sortProblem`, `fieldProblem`).
57
+ * commander takes the next argument as the value of an option that needs one, even when it is
58
+ * a flag, so a forgotten value (`search --fq --rows 5`) silently shifted the rest into the
59
+ * query (`fq=--rows`, `q=5`) and still exited 0. No CKAN filter, sort, facet field or tag query
60
+ * starts with `--`; a single `-` (a negated Solr filter, `-organization:x`) stays allowed.
54
61
  */
55
- export function parseOptionValue(value) {
56
- if (value.startsWith("--")) {
57
- throw new InvalidArgumentError("Expected a value, got another option. Give the option its value first.");
58
- }
59
- return parseNonEmpty(value);
62
+ export function parseOptionValue(problem) {
63
+ const parse = parseWith(problem);
64
+ return (value) => {
65
+ if (value.startsWith("--")) {
66
+ throw new InvalidArgumentError("Expected a value, got another option. Give the option its value first.");
67
+ }
68
+ return parse(value);
69
+ };
60
70
  }
61
71
  /**
62
72
  * Wrap a commander value-parser for a single-valued option so that a second
@@ -74,9 +84,10 @@ export function once(parse) {
74
84
  return parse(value);
75
85
  };
76
86
  }
77
- /** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
78
- export function collectNonEmpty(value, previous = []) {
79
- return previous.concat([parseOptionValue(value)]);
87
+ /** commander accumulator for a repeatable option (`--fq`, `--facet`), each value checked by parseOptionValue(problem). */
88
+ export function collectValues(problem) {
89
+ const parse = parseOptionValue(problem);
90
+ return (value, previous = []) => previous.concat([parse(value)]);
80
91
  }
81
92
  /** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
82
93
  export function parseBoundedInt(min, max) {
@@ -105,7 +116,7 @@ export function parseBaseUrl(value) {
105
116
  export function parsePortal(value) {
106
117
  const portal = findPortal(value, PORTALS);
107
118
  if (!portal) {
108
- throw new InvalidArgumentError(`Unknown portal "${cutForMessage(redactUrl(value))}". \`ckan portals\` lists the known ones.`);
119
+ throw new InvalidArgumentError(`Unknown portal ${quoteValue(redactUrl(value))}. \`ckan portals\` lists the known ones.`);
109
120
  }
110
121
  return portal.url;
111
122
  }
@@ -125,81 +136,6 @@ export function toEngineOptions(global) {
125
136
  options.maxResponseBytes = global.maxResponseBytes;
126
137
  return options;
127
138
  }
128
- /**
129
- * Escape the characters JSON.stringify leaves raw although a terminal acts on them.
130
- * It escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is
131
- * the 8-bit form of CSI) or the bidi formatting characters (isBidiControl), which
132
- * reorder the text that follows. The output is server data, so escape them; the
133
- * result is equivalent, valid JSON (these characters only occur inside strings).
134
- * Checked by char code so the source stays free of control bytes.
135
- */
136
- export function escapeControlChars(json) {
137
- let result = "";
138
- let from = 0;
139
- for (let i = 0; i < json.length; i++) {
140
- const c = json.charCodeAt(i);
141
- if ((c >= 0x7f && c <= 0x9f) || isBidiControl(c)) {
142
- result += json.slice(from, i) + "\\u" + c.toString(16).padStart(4, "0");
143
- from = i + 1;
144
- }
145
- }
146
- return from === 0 ? json : result + json.slice(from);
147
- }
148
- /**
149
- * JSON.stringify, pretty or compact. A deeply nested value (a hostile or broken
150
- * response) overflows the stack — the pretty form far sooner than the compact one,
151
- * which is why the message suggests --compact. The RangeError becomes a CkanError so
152
- * the CLI prints a clear message instead of "Unexpected error: Maximum call stack
153
- * size exceeded".
154
- *
155
- * Compact output takes V8's fast path (no gap) unless the answer holds a huge string: on
156
- * Node 26 the fast path aborts the process ("Zone Allocation failed", exit 134, a native
157
- * dump on stderr) on a string of some 250 MB that JSON.parse made, while the gap path (an
158
- * empty gap, the same text) prints it. The fast path stays the default because on Node 26
159
- * only it is iterative and prints nesting the gap path refuses (~7,000 levels).
160
- */
161
- function stringifyJson(value, compact) {
162
- try {
163
- if (!compact)
164
- return JSON.stringify(value, null, 2);
165
- return holdsHugeString(value) ? JSON.stringify(value, null, "") : JSON.stringify(value);
166
- }
167
- catch (err) {
168
- if (err instanceof RangeError) {
169
- throw new CkanError(compact
170
- ? "The response is nested too deeply to print."
171
- : "The response is nested too deeply to pretty-print; try --compact.", { cause: err });
172
- }
173
- throw err;
174
- }
175
- }
176
- /** A string or key this long sends --compact through the gap path: a quarter of where Node 26's fast path aborts. */
177
- const HUGE_STRING_LENGTH = 64 * 1024 * 1024;
178
- /** Whether `value` holds a string or key of HUGE_STRING_LENGTH or more. Iterative, so nesting costs no stack. */
179
- function holdsHugeString(value) {
180
- const stack = [value];
181
- while (stack.length > 0) {
182
- const item = stack.pop();
183
- if (typeof item === "string") {
184
- if (item.length >= HUGE_STRING_LENGTH)
185
- return true;
186
- }
187
- else if (typeof item === "object" && item !== null) {
188
- for (const [key, element] of Object.entries(item)) {
189
- if (key.length >= HUGE_STRING_LENGTH)
190
- return true;
191
- if (typeof element === "string" || (typeof element === "object" && element !== null))
192
- stack.push(element);
193
- }
194
- }
195
- }
196
- return false;
197
- }
198
- /** Render a JSON value to stdout, pretty by default, compact with --compact. */
199
- export function renderJson(deps, global, value) {
200
- const text = escapeControlChars(stringifyJson(value, global.compact === true));
201
- deps.io.out(text);
202
- }
203
139
  /**
204
140
  * Log a warning (`ckan.http`) when the effective base URL (--portal >
205
141
  * --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
@@ -241,6 +177,8 @@ export function action(deps, fn) {
241
177
  warnOnCleartext(deps, global);
242
178
  const options = toEngineOptions(global);
243
179
  options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
180
+ // An answer delivered changed (invalid text replaced with U+FFFD) is never silent.
181
+ options.onWarning = (message) => logOf(deps).warn("api", message);
244
182
  const client = deps.createClient(options);
245
183
  await fn({ client, global, opts: command.opts() }, positionals);
246
184
  };
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.6.0",
3
+ "version": "0.8.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {