@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.d.ts +0 -7
- package/dist/src/cli/run.js +80 -17
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
2
|
+
import type { CliDeps } from "./io.js";
|
|
3
|
+
import type { GlobalOptions } from "./shared.js";
|
|
4
|
+
/**
|
|
5
|
+
* Printing the answer failed: an answer too deeply nested or too large to print. An ERROR
|
|
6
|
+
* record of `ckan.output`, exit 1.
|
|
7
|
+
*/
|
|
8
|
+
export declare class OutputError extends CkanError {
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Escape the characters JSON.stringify leaves raw although a terminal acts on them. It
|
|
12
|
+
* escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is the 8-bit
|
|
13
|
+
* form of CSI) or the bidi formatting characters (isBidiControl), which reorder the text that
|
|
14
|
+
* follows. The output is server data, so escape them; the result is equivalent, valid JSON
|
|
15
|
+
* (these characters only occur inside strings). Checked by char code so the source stays
|
|
16
|
+
* free of control bytes.
|
|
17
|
+
*
|
|
18
|
+
* Each such character becomes six, so the output is measured first: one longer than V8's
|
|
19
|
+
* longest string is an OutputError ("too large to print"), before anything is built. Then the
|
|
20
|
+
* output is built in chunks of ESCAPE_CHUNK characters, each made flat at once, and joined
|
|
21
|
+
* once: one `+=` per character made a rope of 100 M pieces on an answer of DEL characters, and
|
|
22
|
+
* the process died of a heap out-of-memory crash.
|
|
23
|
+
*/
|
|
24
|
+
export declare function escapeControlChars(json: string): string;
|
|
25
|
+
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
26
|
+
export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
// The output sink: everything the CLI writes as data, to stdout. One place for the rules the
|
|
2
|
+
// maschinenlesbar.org CLIs learned one bug at a time: terminal controls escaped, a deep or huge
|
|
3
|
+
// answer printed (or refused) cleanly, as an ERROR record of `ckan.output`.
|
|
4
|
+
import { constants } from "node:buffer";
|
|
5
|
+
import { isBidiControl } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
6
|
+
import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
7
|
+
/**
|
|
8
|
+
* Printing the answer failed: an answer too deeply nested or too large to print. An ERROR
|
|
9
|
+
* record of `ckan.output`, exit 1.
|
|
10
|
+
*/
|
|
11
|
+
export class OutputError extends CkanError {
|
|
12
|
+
}
|
|
13
|
+
/** The longest string V8 makes (2^29 - 24 characters on 64-bit): a longer JSON text can't be printed at once. */
|
|
14
|
+
const MAX_TEXT_LENGTH = constants.MAX_STRING_LENGTH;
|
|
15
|
+
/**
|
|
16
|
+
* JSON.stringify, pretty or compact. Printing can fail two ways, both an OutputError (exit 1)
|
|
17
|
+
* that names the cause, never "Unexpected error":
|
|
18
|
+
* - an answer nested too deeply (hostile or broken) overflows the stack: "nested too deeply";
|
|
19
|
+
* - a text longer than V8's longest string ("Invalid string length"): "too large to print".
|
|
20
|
+
* "try --compact" is added only when the compact form of this very answer prints: it is probed,
|
|
21
|
+
* not assumed, because whether compact output survives deep nesting depends on the Node version
|
|
22
|
+
* (on Node 26 only the fast path is iterative) and on the machine's stack.
|
|
23
|
+
*
|
|
24
|
+
* Compact output takes V8's fast path (no gap) unless the answer holds a huge string: on Node 26
|
|
25
|
+
* the fast path aborts the process ("Zone Allocation failed", exit 134, a native dump on stderr)
|
|
26
|
+
* on a string of some 250 MB that JSON.parse made, while the gap path (an empty gap, the same
|
|
27
|
+
* text) prints it. The fast path stays the default because on Node 26 only it is iterative and
|
|
28
|
+
* prints nesting the gap path refuses (~7,000 levels).
|
|
29
|
+
*/
|
|
30
|
+
function stringifyJson(value, compact) {
|
|
31
|
+
try {
|
|
32
|
+
return compact ? stringifyCompact(value) : JSON.stringify(value, null, 2);
|
|
33
|
+
}
|
|
34
|
+
catch (err) {
|
|
35
|
+
if (!(err instanceof RangeError))
|
|
36
|
+
throw err;
|
|
37
|
+
const deep = /call stack/i.test(err.message);
|
|
38
|
+
const cause = deep ? "nested too deeply" : `too large (more than ${MAX_TEXT_LENGTH} characters of JSON)`;
|
|
39
|
+
const hint = !compact && compactPrints(value) ? "; try --compact" : "";
|
|
40
|
+
throw new OutputError(`The answer is ${cause} to ${compact ? "print" : "pretty-print"}${hint}.`, { cause: err });
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/** Compact JSON: the fast path, or the gap path for an answer holding a huge string. */
|
|
44
|
+
function stringifyCompact(value) {
|
|
45
|
+
return holdsHugeString(value) ? JSON.stringify(value, null, "") : JSON.stringify(value);
|
|
46
|
+
}
|
|
47
|
+
/** Whether `--compact` prints this answer on this runtime (the hint after a failed pretty-print). */
|
|
48
|
+
function compactPrints(value) {
|
|
49
|
+
try {
|
|
50
|
+
return stringifyCompact(value).length <= MAX_TEXT_LENGTH;
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
/** A string or key this long sends --compact through the gap path: a quarter of where Node 26's fast path aborts. */
|
|
57
|
+
const HUGE_STRING_LENGTH = 64 * 1024 * 1024;
|
|
58
|
+
/**
|
|
59
|
+
* Whether `value` holds a string or key of HUGE_STRING_LENGTH or more. Iterative, so nesting
|
|
60
|
+
* costs no stack, linear, and without an allocation per value: an array is walked by index, an
|
|
61
|
+
* object by `for…in` (own keys only: an answer's objects have none inherited), and only nested
|
|
62
|
+
* objects and arrays are kept for later. (`Object.entries` made a pair per element: on an array
|
|
63
|
+
* of 48M numbers that took 7 GB and crashed.)
|
|
64
|
+
*/
|
|
65
|
+
function holdsHugeString(value) {
|
|
66
|
+
if (typeof value === "string")
|
|
67
|
+
return value.length >= HUGE_STRING_LENGTH;
|
|
68
|
+
const pending = [];
|
|
69
|
+
if (typeof value === "object" && value !== null)
|
|
70
|
+
pending.push(value);
|
|
71
|
+
for (let item = pending.pop(); item !== undefined; item = pending.pop()) {
|
|
72
|
+
if (Array.isArray(item)) {
|
|
73
|
+
for (let i = 0; i < item.length; i++) {
|
|
74
|
+
const element = item[i];
|
|
75
|
+
if (typeof element === "string") {
|
|
76
|
+
if (element.length >= HUGE_STRING_LENGTH)
|
|
77
|
+
return true;
|
|
78
|
+
}
|
|
79
|
+
else if (typeof element === "object" && element !== null)
|
|
80
|
+
pending.push(element);
|
|
81
|
+
}
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
84
|
+
const record = item;
|
|
85
|
+
for (const key in record) {
|
|
86
|
+
if (key.length >= HUGE_STRING_LENGTH)
|
|
87
|
+
return true;
|
|
88
|
+
const element = record[key];
|
|
89
|
+
if (typeof element === "string") {
|
|
90
|
+
if (element.length >= HUGE_STRING_LENGTH)
|
|
91
|
+
return true;
|
|
92
|
+
}
|
|
93
|
+
else if (typeof element === "object" && element !== null)
|
|
94
|
+
pending.push(element);
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
return false;
|
|
98
|
+
}
|
|
99
|
+
/** True for a UTF-16 unit `escapeControlChars` writes as `\uXXXX`. */
|
|
100
|
+
function escaped(c) {
|
|
101
|
+
return (c >= 0x7f && c <= 0x9f) || isBidiControl(c);
|
|
102
|
+
}
|
|
103
|
+
/** How many input characters `escapeControlChars` turns into one piece of output at a time. */
|
|
104
|
+
const ESCAPE_CHUNK = 1 << 16;
|
|
105
|
+
/**
|
|
106
|
+
* Escape the characters JSON.stringify leaves raw although a terminal acts on them. It
|
|
107
|
+
* escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is the 8-bit
|
|
108
|
+
* form of CSI) or the bidi formatting characters (isBidiControl), which reorder the text that
|
|
109
|
+
* follows. The output is server data, so escape them; the result is equivalent, valid JSON
|
|
110
|
+
* (these characters only occur inside strings). Checked by char code so the source stays
|
|
111
|
+
* free of control bytes.
|
|
112
|
+
*
|
|
113
|
+
* Each such character becomes six, so the output is measured first: one longer than V8's
|
|
114
|
+
* longest string is an OutputError ("too large to print"), before anything is built. Then the
|
|
115
|
+
* output is built in chunks of ESCAPE_CHUNK characters, each made flat at once, and joined
|
|
116
|
+
* once: one `+=` per character made a rope of 100 M pieces on an answer of DEL characters, and
|
|
117
|
+
* the process died of a heap out-of-memory crash.
|
|
118
|
+
*/
|
|
119
|
+
export function escapeControlChars(json) {
|
|
120
|
+
let count = 0;
|
|
121
|
+
for (let i = 0; i < json.length; i++)
|
|
122
|
+
if (escaped(json.charCodeAt(i)))
|
|
123
|
+
count++;
|
|
124
|
+
if (count === 0)
|
|
125
|
+
return json;
|
|
126
|
+
const length = json.length + 5 * count;
|
|
127
|
+
if (length > MAX_TEXT_LENGTH) {
|
|
128
|
+
throw new OutputError(`The answer is too large to print (${length} characters once its control characters are escaped).`);
|
|
129
|
+
}
|
|
130
|
+
const chunks = [];
|
|
131
|
+
for (let start = 0; start < json.length; start += ESCAPE_CHUNK) {
|
|
132
|
+
const end = Math.min(start + ESCAPE_CHUNK, json.length);
|
|
133
|
+
const parts = [];
|
|
134
|
+
let from = start;
|
|
135
|
+
for (let i = start; i < end; i++) {
|
|
136
|
+
const c = json.charCodeAt(i);
|
|
137
|
+
if (!escaped(c))
|
|
138
|
+
continue;
|
|
139
|
+
parts.push(json.slice(from, i), "\\u" + c.toString(16).padStart(4, "0"));
|
|
140
|
+
from = i + 1;
|
|
141
|
+
}
|
|
142
|
+
parts.push(json.slice(from, end));
|
|
143
|
+
chunks.push(parts.join(""));
|
|
144
|
+
}
|
|
145
|
+
return chunks.join("");
|
|
146
|
+
}
|
|
147
|
+
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
148
|
+
export function renderJson(deps, global, value) {
|
|
149
|
+
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
150
|
+
deps.io.out(text);
|
|
151
|
+
}
|
package/dist/src/cli/run.d.ts
CHANGED
|
@@ -1,13 +1,6 @@
|
|
|
1
1
|
import { type Command } from "commander";
|
|
2
2
|
import { type CliDeps } from "./io.js";
|
|
3
3
|
import { type Logger } from "./log.js";
|
|
4
|
-
/**
|
|
5
|
-
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
6
|
-
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
7
|
-
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
8
|
-
* `withRedactedOutput`.
|
|
9
|
-
*/
|
|
10
|
-
export declare function redactUserinfo(text: string): string;
|
|
11
4
|
/** The secrets of a run, and the two ways they are replaced. */
|
|
12
5
|
export interface Redaction {
|
|
13
6
|
/**
|
package/dist/src/cli/run.js
CHANGED
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
import { CommanderError } from "commander";
|
|
5
5
|
import { buildProgram, defaultDeps } from "./program.js";
|
|
6
6
|
import { logOf } from "./io.js";
|
|
7
|
+
import { OutputError } from "./output.js";
|
|
7
8
|
import { createLogger, logFormatFromArgv } from "./log.js";
|
|
8
|
-
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
9
|
+
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, redactUrlUserinfo, typedUrlCredentials, } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
9
10
|
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
10
11
|
const USAGE_EXIT = 1;
|
|
11
12
|
/**
|
|
@@ -64,7 +65,7 @@ function writeCommanderErr(command, deps, state, str) {
|
|
|
64
65
|
return;
|
|
65
66
|
if (text.startsWith("error: ")) {
|
|
66
67
|
state.errorLogged = true;
|
|
67
|
-
log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
|
|
68
|
+
log.error("cli", escapeCommanderError(text.slice("error: ".length)).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
|
|
68
69
|
return;
|
|
69
70
|
}
|
|
70
71
|
if (!state.errorLogged) {
|
|
@@ -76,13 +77,35 @@ function writeCommanderErr(command, deps, state, str) {
|
|
|
76
77
|
log.info("cli", line.trimEnd());
|
|
77
78
|
}
|
|
78
79
|
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
*
|
|
80
|
+
* Escape the characters a terminal acts on in one commander error message: CR and LF
|
|
81
|
+
* become `\r` and `\n` (as a log record writes them, `escapeForRecord`), the other C0
|
|
82
|
+
* controls (tab excepted), DEL, C1 and Unicode format characters (bidi overrides,
|
|
83
|
+
* zero-width characters) `\uXXXX`. Commander repeats a rejected argument raw
|
|
84
|
+
* ("command-argument value '<value>' is invalid"), and a value can come from a
|
|
85
|
+
* pipeline: a newline in it could forge a record, an override reorder the line. The
|
|
86
|
+
* record escapes most of these too; this keeps the invisible format characters visible,
|
|
87
|
+
* as the library's own refusals show them (`quoteValue`).
|
|
83
88
|
*/
|
|
84
|
-
|
|
85
|
-
return text.replace(
|
|
89
|
+
function escapeTerminalText(text) {
|
|
90
|
+
return text.replace(/[\u0000-\u0008\u000a-\u001f\u007f-\u009f]|\p{Cf}/gu, (ch) => ch === "\n"
|
|
91
|
+
? "\\n"
|
|
92
|
+
: ch === "\r"
|
|
93
|
+
? "\\r"
|
|
94
|
+
: Array.from({ length: ch.length }, (_, i) => `\\u${ch.charCodeAt(i).toString(16).padStart(4, "0")}`).join(""));
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* escapeTerminalText for one commander error message, except for the
|
|
98
|
+
* `\n(Did you mean …?)` line commander appends after an unknown command or option.
|
|
99
|
+
* That hint is built from this CLI's own command and option names, and it can only
|
|
100
|
+
* end the message: commander quotes the user's value (`unknown command '<value>'`), so
|
|
101
|
+
* a value that contains the same text is followed by its closing quote and stays
|
|
102
|
+
* escaped.
|
|
103
|
+
*/
|
|
104
|
+
function escapeCommanderError(message) {
|
|
105
|
+
const hint = /\n\(Did you mean [^\n]*\?\)$/.exec(message);
|
|
106
|
+
if (hint === null)
|
|
107
|
+
return escapeTerminalText(message);
|
|
108
|
+
return escapeTerminalText(message.slice(0, hint.index)) + hint[0];
|
|
86
109
|
}
|
|
87
110
|
/**
|
|
88
111
|
* The options whose value is a base URL (`--portal` takes a portal's URL as well as its
|
|
@@ -103,6 +126,33 @@ function flagValues(argv, flags) {
|
|
|
103
126
|
});
|
|
104
127
|
return found;
|
|
105
128
|
}
|
|
129
|
+
/** The most URLs `urlsWithin` reads inside one argument, after the one it may start with. */
|
|
130
|
+
const MAX_URLS_WITHIN = 64;
|
|
131
|
+
/** A character a URL scheme may hold after its first letter. */
|
|
132
|
+
const SCHEME_CHAR = /[A-Za-z0-9+.-]/;
|
|
133
|
+
/**
|
|
134
|
+
* `value` and the rest of it from each place a URL starts inside it (`<scheme>://`, the scheme
|
|
135
|
+
* starting with a letter), for `credentialsIn`, which reads a URL at the start of its input only.
|
|
136
|
+
* The library sends a text trimmed, so a padded `" https://user:pw@host"`, one after a dash or
|
|
137
|
+
* inside a phrase (`search "elbe https://…"`, `--fq organization:https://…`) reaches the request
|
|
138
|
+
* URL of an error record, a refusal and commander's echo. Each `://` is found with indexOf and its
|
|
139
|
+
* scheme by walking back, never by a pattern retried at every position (quadratic: 40 000 scheme
|
|
140
|
+
* characters took ~3 s); at most MAX_URLS_WITHIN are read from one argument, since each is parsed
|
|
141
|
+
* whole.
|
|
142
|
+
*/
|
|
143
|
+
function urlsWithin(value) {
|
|
144
|
+
const found = [value];
|
|
145
|
+
for (let at = value.indexOf("://"); at !== -1 && found.length <= MAX_URLS_WITHIN; at = value.indexOf("://", at + 3)) {
|
|
146
|
+
let start = at;
|
|
147
|
+
while (start > 0 && SCHEME_CHAR.test(value[start - 1]))
|
|
148
|
+
start--;
|
|
149
|
+
while (start < at && !/[A-Za-z]/.test(value[start]))
|
|
150
|
+
start++;
|
|
151
|
+
if (start > 0 && start < at)
|
|
152
|
+
found.push(value.slice(start));
|
|
153
|
+
}
|
|
154
|
+
return found;
|
|
155
|
+
}
|
|
106
156
|
/**
|
|
107
157
|
* The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
|
|
108
158
|
* its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
|
|
@@ -113,20 +163,31 @@ function flagValues(argv, flags) {
|
|
|
113
163
|
* `/`; the exact strings can. Without credentials the text passes through unchanged.
|
|
114
164
|
*/
|
|
115
165
|
export function redactionFor(argv, env) {
|
|
116
|
-
// An `--option=value` token is echoed as its value alone
|
|
117
|
-
|
|
166
|
+
// An `--option=value` token is echoed as its value alone; a `key=value` pair (`--param
|
|
167
|
+
// id=https://user:pw@host`) carries its value after the "=", which a URL there starts.
|
|
168
|
+
const values = argv.flatMap((token) => {
|
|
169
|
+
const eq = token.indexOf("=");
|
|
170
|
+
if (eq < 0)
|
|
171
|
+
return [token];
|
|
172
|
+
return token.startsWith("-") ? [token.slice(eq + 1)] : [token, token.slice(eq + 1)];
|
|
173
|
+
});
|
|
118
174
|
// A base URL typed without its scheme is read as if it had one (anywhere else a bare
|
|
119
175
|
// `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
|
|
120
|
-
|
|
121
|
-
|
|
176
|
+
// Read as a URL however its scheme is typed: missing, special without "//", or mistyped
|
|
177
|
+
// (`http//u:pw@h`, the colon left out): typedUrlCredentials.
|
|
178
|
+
const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""];
|
|
179
|
+
const sources = [...values.flatMap(urlsWithin).map(credentialsIn), ...baseUrls.map(typedUrlCredentials)];
|
|
122
180
|
const secrets = new Set();
|
|
123
181
|
const encoded = new Set();
|
|
124
182
|
const echoed = new Set();
|
|
125
183
|
const passwords = new Set();
|
|
126
|
-
for (const
|
|
127
|
-
for (const secret of
|
|
184
|
+
for (const found of sources) {
|
|
185
|
+
for (const secret of found) {
|
|
128
186
|
secrets.add(secret);
|
|
129
187
|
secrets.add(JSON.stringify(secret).slice(1, -1));
|
|
188
|
+
// As commander's echo and the library's quoteValue show it: invisible characters escaped.
|
|
189
|
+
secrets.add(escapeTerminalText(secret));
|
|
190
|
+
secrets.add(escapeTerminalText(JSON.stringify(secret).slice(1, -1)));
|
|
130
191
|
// A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
|
|
131
192
|
// 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
|
|
132
193
|
encoded.add(encodeURIComponent(`${secret}@`));
|
|
@@ -148,7 +209,7 @@ export function redactionFor(argv, env) {
|
|
|
148
209
|
const echoedList = [...echoed].sort((a, b) => b.length - a.length);
|
|
149
210
|
const passwordList = [...passwords].sort((a, b) => b.length - a.length);
|
|
150
211
|
const out = (text) => {
|
|
151
|
-
let result =
|
|
212
|
+
let result = redactUrlUserinfo(redactCredentials(text, list));
|
|
152
213
|
for (const form of encoded)
|
|
153
214
|
result = result.split(form).join("***%40");
|
|
154
215
|
return redactSecrets(result, echoedList);
|
|
@@ -223,10 +284,12 @@ export function processLogger(argv, env = process.env) {
|
|
|
223
284
|
* connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
|
|
224
285
|
* envelope, the wrong result shape, an unknown charset — the portal's answer as much as
|
|
225
286
|
* an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
|
|
226
|
-
* envelope, `CkanActionError`),
|
|
227
|
-
*
|
|
287
|
+
* envelope, `CkanActionError`), `output` for an answer too deeply nested or too large to print
|
|
288
|
+
* (`OutputError`: printing it, not its shape), else `cli`.
|
|
228
289
|
*/
|
|
229
290
|
function areaOf(err) {
|
|
291
|
+
if (err instanceof OutputError)
|
|
292
|
+
return "output";
|
|
230
293
|
if (err instanceof CkanNetworkError)
|
|
231
294
|
return "http";
|
|
232
295
|
if (err instanceof CkanParseError || err instanceof CkanActionError)
|
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { type CliDeps } from "./io.js";
|
|
2
2
|
import { type EngineOptions, type RetryEvent } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
3
|
+
import { type Problem } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
3
4
|
/** commander value-parser: a non-negative integer. */
|
|
4
5
|
export declare function parseIntArg(value: string): number;
|
|
5
|
-
/**
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
export declare function parseNonEmpty(value: string): string;
|
|
6
|
+
/** commander value-parser for free text (the search text): the library's `textProblem`. */
|
|
7
|
+
export declare const parseText: (value: string) => string;
|
|
8
|
+
/** commander value-parser for the id or name of a dataset, resource, organization or group: the library's `idProblem`. */
|
|
9
|
+
export declare const parseId: (value: string) => string;
|
|
10
10
|
/**
|
|
11
11
|
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
|
|
12
12
|
* the library's headerValueProblem rule (not blank, no control characters but tab,
|
|
@@ -14,14 +14,14 @@ export declare function parseNonEmpty(value: string): string;
|
|
|
14
14
|
*/
|
|
15
15
|
export declare function parseHeaderValue(value: string): string;
|
|
16
16
|
/**
|
|
17
|
-
* commander value-parser for
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* 0. No CKAN filter, sort, facet field or tag query
|
|
22
|
-
* (a negated Solr filter, `-organization:x`) stays allowed.
|
|
17
|
+
* A commander value-parser for an option's value: not another long option, then the library's
|
|
18
|
+
* `problem` rule (`textProblem` for a filter or tag query, `sortProblem`, `fieldProblem`).
|
|
19
|
+
* commander takes the next argument as the value of an option that needs one, even when it is
|
|
20
|
+
* a flag, so a forgotten value (`search --fq --rows 5`) silently shifted the rest into the
|
|
21
|
+
* query (`fq=--rows`, `q=5`) and still exited 0. No CKAN filter, sort, facet field or tag query
|
|
22
|
+
* starts with `--`; a single `-` (a negated Solr filter, `-organization:x`) stays allowed.
|
|
23
23
|
*/
|
|
24
|
-
export declare function parseOptionValue(
|
|
24
|
+
export declare function parseOptionValue(problem: Problem<string>): (value: string) => string;
|
|
25
25
|
/**
|
|
26
26
|
* Wrap a commander value-parser for a single-valued option so that a second
|
|
27
27
|
* occurrence is a usage error. commander otherwise keeps only the last value, so
|
|
@@ -30,8 +30,8 @@ export declare function parseOptionValue(value: string): string;
|
|
|
30
30
|
* commander hands the parser as `previous`); build it per program (`buildProgram`).
|
|
31
31
|
*/
|
|
32
32
|
export declare function once<T>(parse: (value: string) => T): (value: string) => T;
|
|
33
|
-
/** commander accumulator for a repeatable option
|
|
34
|
-
export declare function
|
|
33
|
+
/** commander accumulator for a repeatable option (`--fq`, `--facet`), each value checked by parseOptionValue(problem). */
|
|
34
|
+
export declare function collectValues(problem: Problem<string>): (value: string, previous?: string[]) => string[];
|
|
35
35
|
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
36
36
|
export declare function parseBoundedInt(min: number, max: number): (value: string) => number;
|
|
37
37
|
/**
|
|
@@ -55,17 +55,6 @@ export interface GlobalOptions {
|
|
|
55
55
|
}
|
|
56
56
|
/** Translate resolved global CLI options into client EngineOptions. */
|
|
57
57
|
export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
|
|
58
|
-
/**
|
|
59
|
-
* Escape the characters JSON.stringify leaves raw although a terminal acts on them.
|
|
60
|
-
* It escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is
|
|
61
|
-
* the 8-bit form of CSI) or the bidi formatting characters (isBidiControl), which
|
|
62
|
-
* reorder the text that follows. The output is server data, so escape them; the
|
|
63
|
-
* result is equivalent, valid JSON (these characters only occur inside strings).
|
|
64
|
-
* Checked by char code so the source stays free of control bytes.
|
|
65
|
-
*/
|
|
66
|
-
export declare function escapeControlChars(json: string): string;
|
|
67
|
-
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
68
|
-
export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
|
|
69
58
|
/**
|
|
70
59
|
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
71
60
|
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
|
-
// Shared helpers used across CLI command groups: option parsers
|
|
2
|
-
// option resolver
|
|
1
|
+
// Shared helpers used across CLI command groups: option parsers and the global
|
|
2
|
+
// option resolver. Results are written by output.ts.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
4
|
import { logOf } from "./io.js";
|
|
5
|
-
import { cleartextProblem, DEFAULT_BASE_URL
|
|
6
|
-
import {
|
|
5
|
+
import { cleartextProblem, DEFAULT_BASE_URL } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
6
|
+
import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
7
7
|
import { findPortal } from "@maschinenlesbar.org/ckan-client/internal/portals";
|
|
8
|
-
import { baseUrlProblem,
|
|
8
|
+
import { baseUrlProblem, countProblem, headerValueProblem, idProblem, intRangeProblem, textProblem, } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
9
9
|
import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
|
|
10
10
|
/** commander value-parser: a non-negative integer. */
|
|
11
11
|
export function parseIntArg(value) {
|
|
@@ -24,15 +24,22 @@ export function parseIntArg(value) {
|
|
|
24
24
|
return n;
|
|
25
25
|
}
|
|
26
26
|
/**
|
|
27
|
-
* commander value-parser
|
|
28
|
-
*
|
|
27
|
+
* A commander value-parser for the library's `problem` rule (`textProblem`, `idProblem`, …),
|
|
28
|
+
* reported as a usage error before the client is built. The value goes on as typed: the
|
|
29
|
+
* library canonicalises it (`normalizeText`, `normalizeToken`).
|
|
29
30
|
*/
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
31
|
+
function parseWith(problem) {
|
|
32
|
+
return (value) => {
|
|
33
|
+
const reason = problem(value);
|
|
34
|
+
if (reason !== undefined)
|
|
35
|
+
throw new InvalidArgumentError(reason);
|
|
36
|
+
return value;
|
|
37
|
+
};
|
|
35
38
|
}
|
|
39
|
+
/** commander value-parser for free text (the search text): the library's `textProblem`. */
|
|
40
|
+
export const parseText = parseWith(textProblem);
|
|
41
|
+
/** commander value-parser for the id or name of a dataset, resource, organization or group: the library's `idProblem`. */
|
|
42
|
+
export const parseId = parseWith(idProblem);
|
|
36
43
|
/**
|
|
37
44
|
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
|
|
38
45
|
* the library's headerValueProblem rule (not blank, no control characters but tab,
|
|
@@ -45,18 +52,21 @@ export function parseHeaderValue(value) {
|
|
|
45
52
|
return value;
|
|
46
53
|
}
|
|
47
54
|
/**
|
|
48
|
-
* commander value-parser for
|
|
49
|
-
*
|
|
50
|
-
*
|
|
51
|
-
*
|
|
52
|
-
* 0. No CKAN filter, sort, facet field or tag query
|
|
53
|
-
* (a negated Solr filter, `-organization:x`) stays allowed.
|
|
55
|
+
* A commander value-parser for an option's value: not another long option, then the library's
|
|
56
|
+
* `problem` rule (`textProblem` for a filter or tag query, `sortProblem`, `fieldProblem`).
|
|
57
|
+
* commander takes the next argument as the value of an option that needs one, even when it is
|
|
58
|
+
* a flag, so a forgotten value (`search --fq --rows 5`) silently shifted the rest into the
|
|
59
|
+
* query (`fq=--rows`, `q=5`) and still exited 0. No CKAN filter, sort, facet field or tag query
|
|
60
|
+
* starts with `--`; a single `-` (a negated Solr filter, `-organization:x`) stays allowed.
|
|
54
61
|
*/
|
|
55
|
-
export function parseOptionValue(
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
62
|
+
export function parseOptionValue(problem) {
|
|
63
|
+
const parse = parseWith(problem);
|
|
64
|
+
return (value) => {
|
|
65
|
+
if (value.startsWith("--")) {
|
|
66
|
+
throw new InvalidArgumentError("Expected a value, got another option. Give the option its value first.");
|
|
67
|
+
}
|
|
68
|
+
return parse(value);
|
|
69
|
+
};
|
|
60
70
|
}
|
|
61
71
|
/**
|
|
62
72
|
* Wrap a commander value-parser for a single-valued option so that a second
|
|
@@ -74,9 +84,10 @@ export function once(parse) {
|
|
|
74
84
|
return parse(value);
|
|
75
85
|
};
|
|
76
86
|
}
|
|
77
|
-
/** commander accumulator for a repeatable option
|
|
78
|
-
export function
|
|
79
|
-
|
|
87
|
+
/** commander accumulator for a repeatable option (`--fq`, `--facet`), each value checked by parseOptionValue(problem). */
|
|
88
|
+
export function collectValues(problem) {
|
|
89
|
+
const parse = parseOptionValue(problem);
|
|
90
|
+
return (value, previous = []) => previous.concat([parse(value)]);
|
|
80
91
|
}
|
|
81
92
|
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
82
93
|
export function parseBoundedInt(min, max) {
|
|
@@ -105,7 +116,7 @@ export function parseBaseUrl(value) {
|
|
|
105
116
|
export function parsePortal(value) {
|
|
106
117
|
const portal = findPortal(value, PORTALS);
|
|
107
118
|
if (!portal) {
|
|
108
|
-
throw new InvalidArgumentError(`Unknown portal
|
|
119
|
+
throw new InvalidArgumentError(`Unknown portal ${quoteValue(redactUrl(value))}. \`ckan portals\` lists the known ones.`);
|
|
109
120
|
}
|
|
110
121
|
return portal.url;
|
|
111
122
|
}
|
|
@@ -125,81 +136,6 @@ export function toEngineOptions(global) {
|
|
|
125
136
|
options.maxResponseBytes = global.maxResponseBytes;
|
|
126
137
|
return options;
|
|
127
138
|
}
|
|
128
|
-
/**
|
|
129
|
-
* Escape the characters JSON.stringify leaves raw although a terminal acts on them.
|
|
130
|
-
* It escapes C0 (including ESC) but not DEL, the C1 range U+0080–U+009F (U+009B is
|
|
131
|
-
* the 8-bit form of CSI) or the bidi formatting characters (isBidiControl), which
|
|
132
|
-
* reorder the text that follows. The output is server data, so escape them; the
|
|
133
|
-
* result is equivalent, valid JSON (these characters only occur inside strings).
|
|
134
|
-
* Checked by char code so the source stays free of control bytes.
|
|
135
|
-
*/
|
|
136
|
-
export function escapeControlChars(json) {
|
|
137
|
-
let result = "";
|
|
138
|
-
let from = 0;
|
|
139
|
-
for (let i = 0; i < json.length; i++) {
|
|
140
|
-
const c = json.charCodeAt(i);
|
|
141
|
-
if ((c >= 0x7f && c <= 0x9f) || isBidiControl(c)) {
|
|
142
|
-
result += json.slice(from, i) + "\\u" + c.toString(16).padStart(4, "0");
|
|
143
|
-
from = i + 1;
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return from === 0 ? json : result + json.slice(from);
|
|
147
|
-
}
|
|
148
|
-
/**
|
|
149
|
-
* JSON.stringify, pretty or compact. A deeply nested value (a hostile or broken
|
|
150
|
-
* response) overflows the stack — the pretty form far sooner than the compact one,
|
|
151
|
-
* which is why the message suggests --compact. The RangeError becomes a CkanError so
|
|
152
|
-
* the CLI prints a clear message instead of "Unexpected error: Maximum call stack
|
|
153
|
-
* size exceeded".
|
|
154
|
-
*
|
|
155
|
-
* Compact output takes V8's fast path (no gap) unless the answer holds a huge string: on
|
|
156
|
-
* Node 26 the fast path aborts the process ("Zone Allocation failed", exit 134, a native
|
|
157
|
-
* dump on stderr) on a string of some 250 MB that JSON.parse made, while the gap path (an
|
|
158
|
-
* empty gap, the same text) prints it. The fast path stays the default because on Node 26
|
|
159
|
-
* only it is iterative and prints nesting the gap path refuses (~7,000 levels).
|
|
160
|
-
*/
|
|
161
|
-
function stringifyJson(value, compact) {
|
|
162
|
-
try {
|
|
163
|
-
if (!compact)
|
|
164
|
-
return JSON.stringify(value, null, 2);
|
|
165
|
-
return holdsHugeString(value) ? JSON.stringify(value, null, "") : JSON.stringify(value);
|
|
166
|
-
}
|
|
167
|
-
catch (err) {
|
|
168
|
-
if (err instanceof RangeError) {
|
|
169
|
-
throw new CkanError(compact
|
|
170
|
-
? "The response is nested too deeply to print."
|
|
171
|
-
: "The response is nested too deeply to pretty-print; try --compact.", { cause: err });
|
|
172
|
-
}
|
|
173
|
-
throw err;
|
|
174
|
-
}
|
|
175
|
-
}
|
|
176
|
-
/** A string or key this long sends --compact through the gap path: a quarter of where Node 26's fast path aborts. */
|
|
177
|
-
const HUGE_STRING_LENGTH = 64 * 1024 * 1024;
|
|
178
|
-
/** Whether `value` holds a string or key of HUGE_STRING_LENGTH or more. Iterative, so nesting costs no stack. */
|
|
179
|
-
function holdsHugeString(value) {
|
|
180
|
-
const stack = [value];
|
|
181
|
-
while (stack.length > 0) {
|
|
182
|
-
const item = stack.pop();
|
|
183
|
-
if (typeof item === "string") {
|
|
184
|
-
if (item.length >= HUGE_STRING_LENGTH)
|
|
185
|
-
return true;
|
|
186
|
-
}
|
|
187
|
-
else if (typeof item === "object" && item !== null) {
|
|
188
|
-
for (const [key, element] of Object.entries(item)) {
|
|
189
|
-
if (key.length >= HUGE_STRING_LENGTH)
|
|
190
|
-
return true;
|
|
191
|
-
if (typeof element === "string" || (typeof element === "object" && element !== null))
|
|
192
|
-
stack.push(element);
|
|
193
|
-
}
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
return false;
|
|
197
|
-
}
|
|
198
|
-
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
199
|
-
export function renderJson(deps, global, value) {
|
|
200
|
-
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
201
|
-
deps.io.out(text);
|
|
202
|
-
}
|
|
203
139
|
/**
|
|
204
140
|
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
205
141
|
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
@@ -241,6 +177,8 @@ export function action(deps, fn) {
|
|
|
241
177
|
warnOnCleartext(deps, global);
|
|
242
178
|
const options = toEngineOptions(global);
|
|
243
179
|
options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
|
|
180
|
+
// An answer delivered changed (invalid text replaced with U+FFFD) is never silent.
|
|
181
|
+
options.onWarning = (message) => logOf(deps).warn("api", message);
|
|
244
182
|
const client = deps.createClient(options);
|
|
245
183
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
246
184
|
};
|
package/package.json
CHANGED