@maschinenlesbar.org/ckan-cli 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +35 -8
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -6
- package/dist/src/cli/io.js +62 -10
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +72 -2
- package/dist/src/cli/run.d.ts +35 -10
- package/dist/src/cli/run.js +175 -25
- package/dist/src/cli/shared.d.ts +9 -1
- package/dist/src/cli/shared.js +17 -3
- package/dist/src/client/client.js +4 -3
- package/dist/src/client/engine.d.ts +15 -1
- package/dist/src/client/engine.js +51 -8
- package/dist/src/client/errors.d.ts +51 -7
- package/dist/src/client/errors.js +100 -16
- package/dist/src/client/index.d.ts +2 -2
- package/dist/src/client/index.js +2 -2
- package/dist/src/client/portals.js +3 -3
- package/dist/src/client/validate.js +2 -2
- package/package.json +4 -3
|
@@ -3,4 +3,11 @@ import type { CliDeps } from "./io.js";
|
|
|
3
3
|
export declare const VERSION: string;
|
|
4
4
|
/** Default dependencies: real client + real stdout/stderr + real environment. */
|
|
5
5
|
export declare const defaultDeps: CliDeps;
|
|
6
|
+
/**
|
|
7
|
+
* commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
|
|
8
|
+
* or "" when nothing is close: the same rule as commander's own (an optimal-string-
|
|
9
|
+
* alignment distance below 3 that keeps more than 40 % of the word). The root action
|
|
10
|
+
* reports an unknown command itself, so commander's suggestion is rebuilt here.
|
|
11
|
+
*/
|
|
12
|
+
export declare function suggestSimilar(word: string, candidates: readonly string[]): string;
|
|
6
13
|
export declare function buildProgram(deps?: CliDeps): Command;
|
package/dist/src/cli/program.js
CHANGED
|
@@ -8,10 +8,11 @@ import { defaultIO } from "./io.js";
|
|
|
8
8
|
import { CkanClient } from "../client/client.js";
|
|
9
9
|
import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
|
|
10
10
|
import { MAX_TIMEOUT_MS } from "../client/http.js";
|
|
11
|
-
import { redactUrl } from "../client/errors.js";
|
|
11
|
+
import { cutForMessage, redactUrl } from "../client/errors.js";
|
|
12
12
|
import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
|
|
13
13
|
import { registerCatalogueCommands } from "./commands/catalogue.js";
|
|
14
14
|
import { registerPortalCommands } from "./commands/portal.js";
|
|
15
|
+
import { DEFAULT_LOG_FORMAT, logFormatProblem } from "./log.js";
|
|
15
16
|
/**
|
|
16
17
|
* Single source of truth for the version: read from package.json at runtime
|
|
17
18
|
* rather than duplicating a literal that can silently drift after a release bump.
|
|
@@ -35,6 +36,60 @@ export const defaultDeps = {
|
|
|
35
36
|
env: process.env,
|
|
36
37
|
createClient: (options) => new CkanClient(options),
|
|
37
38
|
};
|
|
39
|
+
/** commander value-parser for `--log-format`. */
|
|
40
|
+
function parseLogFormat(value) {
|
|
41
|
+
const problem = logFormatProblem(value);
|
|
42
|
+
if (problem !== undefined)
|
|
43
|
+
throw new InvalidArgumentError(problem);
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
|
|
48
|
+
* or "" when nothing is close: the same rule as commander's own (an optimal-string-
|
|
49
|
+
* alignment distance below 3 that keeps more than 40 % of the word). The root action
|
|
50
|
+
* reports an unknown command itself, so commander's suggestion is rebuilt here.
|
|
51
|
+
*/
|
|
52
|
+
export function suggestSimilar(word, candidates) {
|
|
53
|
+
let similar = [];
|
|
54
|
+
let best = 3;
|
|
55
|
+
for (const candidate of new Set(candidates)) {
|
|
56
|
+
if (candidate.length <= 1)
|
|
57
|
+
continue;
|
|
58
|
+
const distance = editDistance(word, candidate, best);
|
|
59
|
+
const length = Math.max(word.length, candidate.length);
|
|
60
|
+
if ((length - distance) / length <= 0.4)
|
|
61
|
+
continue;
|
|
62
|
+
if (distance < best) {
|
|
63
|
+
best = distance;
|
|
64
|
+
similar = [candidate];
|
|
65
|
+
}
|
|
66
|
+
else if (distance === best) {
|
|
67
|
+
similar.push(candidate);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
similar.sort((a, b) => a.localeCompare(b));
|
|
71
|
+
if (similar.length > 1)
|
|
72
|
+
return `\n(Did you mean one of ${similar.join(", ")}?)`;
|
|
73
|
+
return similar.length === 1 ? `\n(Did you mean ${similar[0]}?)` : "";
|
|
74
|
+
}
|
|
75
|
+
/** Optimal-string-alignment distance between `a` and `b`; at least `cap` when it is that far. */
|
|
76
|
+
function editDistance(a, b, cap) {
|
|
77
|
+
if (Math.abs(a.length - b.length) >= cap)
|
|
78
|
+
return Math.max(a.length, b.length);
|
|
79
|
+
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
|
|
80
|
+
for (let j = 1; j <= b.length; j++)
|
|
81
|
+
d[0][j] = j;
|
|
82
|
+
for (let i = 1; i <= a.length; i++) {
|
|
83
|
+
for (let j = 1; j <= b.length; j++) {
|
|
84
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
85
|
+
let v = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
|
|
86
|
+
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1])
|
|
87
|
+
v = Math.min(v, d[i - 2][j - 2] + 1);
|
|
88
|
+
d[i][j] = v;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return d[a.length][b.length];
|
|
92
|
+
}
|
|
38
93
|
export function buildProgram(deps = defaultDeps) {
|
|
39
94
|
const program = new Command();
|
|
40
95
|
const baseUrlDefault = deps.env["CKAN_BASE_URL"] || DEFAULT_BASE_URL;
|
|
@@ -54,6 +109,7 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
54
109
|
.option("--user-agent <ua>", "User-Agent header value", once(parseHeaderValue))
|
|
55
110
|
.option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each backs off linearly, or waits a longer Retry-After up to ${MAX_RETRY_AFTER_MS / 1000} s)`, once(parseBoundedInt(0, MAX_RETRIES)))
|
|
56
111
|
.option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", once(parseIntArg))
|
|
112
|
+
.option("--log-format <format>", `how errors, warnings and notes are written to stderr: text (log4j style: time, level, [topic], message) or jsonl (one JSON object per line: ts, level, topic, msg); default ${DEFAULT_LOG_FORMAT}`, once(parseLogFormat))
|
|
57
113
|
.option("--compact", "print JSON on a single line instead of pretty-printed")
|
|
58
114
|
.showHelpAfterError();
|
|
59
115
|
// A bare invocation (no subcommand) should print help to stdout and exit 0,
|
|
@@ -68,7 +124,12 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
68
124
|
program.action(() => {
|
|
69
125
|
const [unknown] = program.args;
|
|
70
126
|
if (unknown !== undefined) {
|
|
71
|
-
|
|
127
|
+
// The name is the user's: cut, after its credentials are redacted (a cut could
|
|
128
|
+
// otherwise leave part of a password without the "@" the redaction keys on).
|
|
129
|
+
const names = program.commands.map((c) => c.name()).filter((name) => name !== "help");
|
|
130
|
+
program.error(`error: unknown command '${cutForMessage(redactUrl(unknown))}'${suggestSimilar(unknown, names)}`, {
|
|
131
|
+
code: "commander.unknownCommand",
|
|
132
|
+
});
|
|
72
133
|
}
|
|
73
134
|
program.help();
|
|
74
135
|
});
|
|
@@ -79,6 +140,15 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
79
140
|
// prints the built-in list or checks each portal at its own URL — a user whose
|
|
80
141
|
// variable is broken is exactly the one looking for a working portal id.
|
|
81
142
|
const offline = new Set(["help", "portals"]);
|
|
143
|
+
// One source for the log format once commander has parsed argv: its value, not the
|
|
144
|
+
// scan of argv (an option's value can look like --log-format; `--` ends the scan, not
|
|
145
|
+
// commander's parse of a value). The first hook, so the CKAN_BASE_URL check below
|
|
146
|
+
// already logs in that format.
|
|
147
|
+
program.hook("preAction", (_program, actionCommand) => {
|
|
148
|
+
const format = actionCommand.optsWithGlobals().logFormat;
|
|
149
|
+
if (deps.log !== undefined)
|
|
150
|
+
deps.log.format = format ?? DEFAULT_LOG_FORMAT;
|
|
151
|
+
});
|
|
82
152
|
program.hook("preAction", (_program, actionCommand) => {
|
|
83
153
|
if (actionCommand === program || offline.has(actionCommand.name()))
|
|
84
154
|
return;
|
package/dist/src/cli/run.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type Command } from "commander";
|
|
2
|
-
import type
|
|
2
|
+
import { type CliDeps } from "./io.js";
|
|
3
|
+
import { type Logger } from "./log.js";
|
|
3
4
|
/**
|
|
4
5
|
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
5
6
|
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
@@ -7,16 +8,33 @@ import type { CliDeps } from "./io.js";
|
|
|
7
8
|
* `withRedactedOutput`.
|
|
8
9
|
*/
|
|
9
10
|
export declare function redactUserinfo(text: string): string;
|
|
11
|
+
/** The secrets of a run, and the two ways they are replaced. */
|
|
12
|
+
export interface Redaction {
|
|
13
|
+
/**
|
|
14
|
+
* stdout text: the userinfo of every URL-like argument and of CKAN_BASE_URL replaced
|
|
15
|
+
* (`***@`), and the forms a server echoes it back in (the Basic value, the decoded
|
|
16
|
+
* `user:password`).
|
|
17
|
+
*/
|
|
18
|
+
out(text: string): string;
|
|
19
|
+
/** stderr text, a record's message: that, and the password alone (`***`). */
|
|
20
|
+
err(text: string): string;
|
|
21
|
+
}
|
|
10
22
|
/**
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
|
|
19
|
-
|
|
23
|
+
* The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
|
|
24
|
+
* its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
|
|
25
|
+
* own messages name the values they refuse, and help shows the base URL's default:
|
|
26
|
+
* whatever path a credential takes to stdout or stderr, the exact userinfo (as
|
|
27
|
+
* `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
|
|
28
|
+
* by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
|
|
29
|
+
* `/`; the exact strings can. Without credentials the text passes through unchanged.
|
|
30
|
+
*/
|
|
31
|
+
export declare function redactionFor(argv: readonly string[], env: Record<string, string | undefined>): Redaction;
|
|
32
|
+
/**
|
|
33
|
+
* `deps` that keep the secrets of this run (`redactionFor`) out of everything they
|
|
34
|
+
* print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
|
|
35
|
+
* message before formatting it, then writes to the raw `io.err`, so the frame is never
|
|
36
|
+
* touched and a password with DEL, C1 or bidi characters is matched in its raw form.
|
|
37
|
+
* `io.err` itself is redacted too, for anything that writes to stderr without the log.
|
|
20
38
|
*/
|
|
21
39
|
export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[]): CliDeps;
|
|
22
40
|
/**
|
|
@@ -26,4 +44,11 @@ export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[
|
|
|
26
44
|
* values; scanning stops at `--`.
|
|
27
45
|
*/
|
|
28
46
|
export declare function unknownHelpTopic(program: Command, argv: readonly string[]): number | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* The log for what happens outside `run()`, in the bin shim: a stdout write error
|
|
49
|
+
* (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
|
|
50
|
+
* for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
|
|
51
|
+
* run's own log; it writes to the raw stderr.
|
|
52
|
+
*/
|
|
53
|
+
export declare function processLogger(argv: readonly string[], env?: Record<string, string | undefined>): Logger;
|
|
29
54
|
export declare function run(argv: string[], deps?: CliDeps): Promise<number>;
|
package/dist/src/cli/run.js
CHANGED
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
// captured output and exit code without spawning a subprocess.
|
|
4
4
|
import { CommanderError } from "commander";
|
|
5
5
|
import { buildProgram, defaultDeps } from "./program.js";
|
|
6
|
-
import {
|
|
6
|
+
import { logOf } from "./io.js";
|
|
7
|
+
import { createLogger, logFormatFromArgv } from "./log.js";
|
|
8
|
+
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "../client/errors.js";
|
|
7
9
|
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
8
10
|
const USAGE_EXIT = 1;
|
|
9
11
|
/**
|
|
@@ -11,14 +13,67 @@ const USAGE_EXIT = 1;
|
|
|
11
13
|
* commander does not propagate these to subcommands, so a parse error on a
|
|
12
14
|
* subcommand would otherwise call process.exit() and bypass our error handling.
|
|
13
15
|
*/
|
|
14
|
-
function configureTree(command, deps) {
|
|
16
|
+
function configureTree(command, deps, state = { errorLogged: false }) {
|
|
15
17
|
command.exitOverride();
|
|
16
18
|
command.configureOutput({
|
|
17
19
|
writeOut: (str) => deps.io.out(str.replace(/\n$/, "")),
|
|
18
|
-
writeErr: (str) => deps
|
|
20
|
+
writeErr: (str) => writeCommanderErr(command, deps, state, str),
|
|
19
21
|
});
|
|
20
22
|
for (const child of command.commands)
|
|
21
|
-
configureTree(child, deps);
|
|
23
|
+
configureTree(child, deps, state);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The names (long and short) of the program's own options that require a value
|
|
27
|
+
* (`--user-agent`). Only the program's: commander takes them out of argv wherever they
|
|
28
|
+
* stand, before a subcommand sees the rest, so a subcommand's `--rows` never swallows
|
|
29
|
+
* a `--log-format` after it.
|
|
30
|
+
*/
|
|
31
|
+
function valueOptionsOf(program) {
|
|
32
|
+
const names = new Set();
|
|
33
|
+
for (const option of program.options) {
|
|
34
|
+
if (!option.required)
|
|
35
|
+
continue;
|
|
36
|
+
if (option.long !== undefined)
|
|
37
|
+
names.add(option.long);
|
|
38
|
+
if (option.short !== undefined)
|
|
39
|
+
names.add(option.short);
|
|
40
|
+
}
|
|
41
|
+
return names;
|
|
42
|
+
}
|
|
43
|
+
/** `ckan search`: the command's name with its parents'. */
|
|
44
|
+
function commandPath(command) {
|
|
45
|
+
const names = [];
|
|
46
|
+
for (let c = command; c !== null; c = c.parent)
|
|
47
|
+
names.unshift(c.name());
|
|
48
|
+
return names.join(" ");
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* commander's stderr output as log records, one per line. Its `error: …` is an ERROR of
|
|
52
|
+
* `cli`, with a following `(Did you mean …?)` line appended to that same record; the
|
|
53
|
+
* help it shows after an error is one INFO record per non-blank line. A command group
|
|
54
|
+
* run without its subcommand makes commander show the help as an error with no
|
|
55
|
+
* `error:` line: an ERROR record "missing command: `ckan <group> <subcommand>`" comes
|
|
56
|
+
* first, so every failed run has one (ckan has no groups today; a bare `ckan` prints
|
|
57
|
+
* its help to stdout and exits 0).
|
|
58
|
+
*/
|
|
59
|
+
function writeCommanderErr(command, deps, state, str) {
|
|
60
|
+
const log = logOf(deps);
|
|
61
|
+
const text = str.replace(/\n$/, "");
|
|
62
|
+
// The blank line commander writes between an error and the help it shows after.
|
|
63
|
+
if (text.trim() === "")
|
|
64
|
+
return;
|
|
65
|
+
if (text.startsWith("error: ")) {
|
|
66
|
+
state.errorLogged = true;
|
|
67
|
+
log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (!state.errorLogged) {
|
|
71
|
+
state.errorLogged = true;
|
|
72
|
+
log.error("cli", `missing command: \`${commandPath(command)} <subcommand>\``);
|
|
73
|
+
}
|
|
74
|
+
for (const line of text.split("\n"))
|
|
75
|
+
if (line.trim() !== "")
|
|
76
|
+
log.info("cli", line.trimEnd());
|
|
22
77
|
}
|
|
23
78
|
/**
|
|
24
79
|
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
@@ -30,22 +85,44 @@ export function redactUserinfo(text) {
|
|
|
30
85
|
return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
|
|
31
86
|
}
|
|
32
87
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
* client's own messages name the values they refuse, and help shows the base URL's
|
|
37
|
-
* default: whatever path a credential takes to stdout or stderr, the exact userinfo
|
|
38
|
-
* (as `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is
|
|
39
|
-
* replaced by `***`. A pattern alone can't delimit a password with spaces, quotes,
|
|
40
|
-
* `#`, `?` or `/`; the exact strings can. Without credentials the output passes
|
|
41
|
-
* through unchanged.
|
|
88
|
+
* The options whose value is a base URL (`--portal` takes a portal's URL as well as its
|
|
89
|
+
* id): a `user:password@host` given there without its scheme is still a credential, like
|
|
90
|
+
* one in CKAN_BASE_URL (anywhere else a bare `a:b@c` is not).
|
|
42
91
|
*/
|
|
43
|
-
|
|
92
|
+
const BASE_URL_FLAGS = ["--base-url", "--portal"];
|
|
93
|
+
/** The values of the `flags` in `argv`, in both forms (`--flag value`, `--flag=value`). */
|
|
94
|
+
function flagValues(argv, flags) {
|
|
95
|
+
const found = [];
|
|
96
|
+
argv.forEach((token, i) => {
|
|
97
|
+
const next = argv[i + 1];
|
|
98
|
+
if (flags.includes(token) && next !== undefined)
|
|
99
|
+
found.push(next);
|
|
100
|
+
const eq = token.indexOf("=");
|
|
101
|
+
if (eq > 0 && flags.includes(token.slice(0, eq)))
|
|
102
|
+
found.push(token.slice(eq + 1));
|
|
103
|
+
});
|
|
104
|
+
return found;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
|
|
108
|
+
* its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
|
|
109
|
+
* own messages name the values they refuse, and help shows the base URL's default:
|
|
110
|
+
* whatever path a credential takes to stdout or stderr, the exact userinfo (as
|
|
111
|
+
* `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
|
|
112
|
+
* by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
|
|
113
|
+
* `/`; the exact strings can. Without credentials the text passes through unchanged.
|
|
114
|
+
*/
|
|
115
|
+
export function redactionFor(argv, env) {
|
|
44
116
|
// An `--option=value` token is echoed as its value alone.
|
|
45
117
|
const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
|
|
46
|
-
|
|
118
|
+
// A base URL typed without its scheme is read as if it had one (anywhere else a bare
|
|
119
|
+
// `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
|
|
120
|
+
const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
|
|
121
|
+
const sources = [...values, ...baseUrls];
|
|
47
122
|
const secrets = new Set();
|
|
48
123
|
const encoded = new Set();
|
|
124
|
+
const echoed = new Set();
|
|
125
|
+
const passwords = new Set();
|
|
49
126
|
for (const source of sources) {
|
|
50
127
|
for (const secret of credentialsIn(source)) {
|
|
51
128
|
secrets.add(secret);
|
|
@@ -53,18 +130,51 @@ export function withRedactedOutput(deps, argv) {
|
|
|
53
130
|
// A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
|
|
54
131
|
// 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
|
|
55
132
|
encoded.add(encodeURIComponent(`${secret}@`));
|
|
133
|
+
// What a server echoes back: the Basic value and the decoded user:password on
|
|
134
|
+
// stdout and stderr, the password alone (it may well occur in the data) on stderr.
|
|
135
|
+
const [basic, pair, password] = echoedCredentialForms(secret);
|
|
136
|
+
if (basic !== undefined)
|
|
137
|
+
echoed.add(basic);
|
|
138
|
+
if (pair !== undefined)
|
|
139
|
+
echoed.add(pair);
|
|
140
|
+
if (password !== undefined)
|
|
141
|
+
passwords.add(password);
|
|
56
142
|
}
|
|
57
143
|
}
|
|
58
144
|
if (secrets.size === 0)
|
|
59
|
-
return
|
|
145
|
+
return { out: (text) => text, err: (text) => text };
|
|
60
146
|
const list = [...secrets];
|
|
61
|
-
|
|
62
|
-
|
|
147
|
+
// Longest first, so a secret is never left half-replaced by one of its own substrings.
|
|
148
|
+
const echoedList = [...echoed].sort((a, b) => b.length - a.length);
|
|
149
|
+
const passwordList = [...passwords].sort((a, b) => b.length - a.length);
|
|
150
|
+
const out = (text) => {
|
|
151
|
+
let result = redactUserinfo(redactCredentials(text, list));
|
|
63
152
|
for (const form of encoded)
|
|
64
|
-
|
|
65
|
-
return
|
|
153
|
+
result = result.split(form).join("***%40");
|
|
154
|
+
return redactSecrets(result, echoedList);
|
|
155
|
+
};
|
|
156
|
+
return { out, err: (text) => redactSecrets(out(text), passwordList) };
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* `deps` that keep the secrets of this run (`redactionFor`) out of everything they
|
|
160
|
+
* print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
|
|
161
|
+
* message before formatting it, then writes to the raw `io.err`, so the frame is never
|
|
162
|
+
* touched and a password with DEL, C1 or bidi characters is matched in its raw form.
|
|
163
|
+
* `io.err` itself is redacted too, for anything that writes to stderr without the log.
|
|
164
|
+
*/
|
|
165
|
+
export function withRedactedOutput(deps, argv) {
|
|
166
|
+
const redaction = redactionFor(argv, deps.env);
|
|
167
|
+
const { out, err } = deps.io;
|
|
168
|
+
return {
|
|
169
|
+
...deps,
|
|
170
|
+
io: { ...deps.io, out: (text) => out(redaction.out(text)), err: (text) => err(redaction.err(text)) },
|
|
171
|
+
log: createLogger({
|
|
172
|
+
format: logFormatFromArgv(argv),
|
|
173
|
+
write: err,
|
|
174
|
+
redact: redaction.err,
|
|
175
|
+
...(deps.now === undefined ? {} : { now: deps.now }),
|
|
176
|
+
}),
|
|
66
177
|
};
|
|
67
|
-
return { ...deps, io: { out: (text) => deps.io.out(redact(text)), err: (text) => deps.io.err(redact(text)) } };
|
|
68
178
|
}
|
|
69
179
|
/**
|
|
70
180
|
* The index of a `help` command whose topic names no command (`ckan help nonexistent`),
|
|
@@ -95,10 +205,44 @@ export function unknownHelpTopic(program, argv) {
|
|
|
95
205
|
}
|
|
96
206
|
return undefined;
|
|
97
207
|
}
|
|
208
|
+
/**
|
|
209
|
+
* The log for what happens outside `run()`, in the bin shim: a stdout write error
|
|
210
|
+
* (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
|
|
211
|
+
* for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
|
|
212
|
+
* run's own log; it writes to the raw stderr.
|
|
213
|
+
*/
|
|
214
|
+
export function processLogger(argv, env = process.env) {
|
|
215
|
+
return createLogger({
|
|
216
|
+
format: logFormatFromArgv(argv),
|
|
217
|
+
write: (line) => process.stderr.write(line + "\n"),
|
|
218
|
+
redact: redactionFor(argv, env).err,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* The log area of a `CkanError` that is neither an API error nor a usage error: the
|
|
223
|
+
* connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
|
|
224
|
+
* envelope, the wrong result shape, an unknown charset — the portal's answer as much as
|
|
225
|
+
* an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
|
|
226
|
+
* envelope, `CkanActionError`), else `cli` (a response nested too deeply to print is
|
|
227
|
+
* about printing the answer, not its shape).
|
|
228
|
+
*/
|
|
229
|
+
function areaOf(err) {
|
|
230
|
+
if (err instanceof CkanNetworkError)
|
|
231
|
+
return "http";
|
|
232
|
+
if (err instanceof CkanParseError || err instanceof CkanActionError)
|
|
233
|
+
return "api";
|
|
234
|
+
return "cli";
|
|
235
|
+
}
|
|
98
236
|
export async function run(argv, deps = defaultDeps) {
|
|
237
|
+
// The log replaces the secrets of the run in every message, in either format.
|
|
99
238
|
deps = withRedactedOutput(deps, argv);
|
|
100
239
|
const program = buildProgram(deps);
|
|
101
240
|
configureTree(program, deps);
|
|
241
|
+
// For the records of a parse error: the scan of argv, now knowing which of the program's
|
|
242
|
+
// options take a value, as commander reads them. Once commander has parsed argv, the program's
|
|
243
|
+
// first preAction hook sets the format it parsed (`buildProgram`).
|
|
244
|
+
if (deps.log !== undefined)
|
|
245
|
+
deps.log.format = logFormatFromArgv(argv, valueOptionsOf(program));
|
|
102
246
|
// `help <unknown>` is answered like `<unknown>`: "error: unknown command '<unknown>'",
|
|
103
247
|
// the help after it, exit 1 — not the help alone with a failure code.
|
|
104
248
|
const helpAt = unknownHelpTopic(program, argv);
|
|
@@ -113,24 +257,30 @@ export async function run(argv, deps = defaultDeps) {
|
|
|
113
257
|
// Help/version requests exit 0; genuine parse errors carry their own code.
|
|
114
258
|
return err.exitCode;
|
|
115
259
|
}
|
|
260
|
+
const log = logOf(deps);
|
|
116
261
|
if (err instanceof CkanValidationError) {
|
|
117
262
|
// An input the library refused before any request (a rule the commander
|
|
118
263
|
// parsers do not see, e.g. one across parameters): a usage error.
|
|
119
|
-
|
|
264
|
+
log.error("cli", err.message);
|
|
120
265
|
return USAGE_EXIT;
|
|
121
266
|
}
|
|
122
267
|
if (err instanceof CkanApiError) {
|
|
123
|
-
|
|
268
|
+
log.error("api", err.message);
|
|
124
269
|
// Map a few notable statuses to distinct exit codes for scripting.
|
|
125
270
|
if (err.status === 404)
|
|
126
271
|
return 4;
|
|
127
272
|
return 1;
|
|
128
273
|
}
|
|
274
|
+
if (err instanceof CkanActionError) {
|
|
275
|
+
log.error(areaOf(err), err.message);
|
|
276
|
+
// CKAN's Not Found Error is a not-found whatever the HTTP status it came with.
|
|
277
|
+
return err.isNotFound ? 4 : 1;
|
|
278
|
+
}
|
|
129
279
|
if (err instanceof CkanError) {
|
|
130
|
-
|
|
280
|
+
log.error(areaOf(err), err.message);
|
|
131
281
|
return 1;
|
|
132
282
|
}
|
|
133
|
-
|
|
283
|
+
log.error("cli", `Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
|
|
134
284
|
return 1;
|
|
135
285
|
}
|
|
136
286
|
}
|
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type CliDeps } from "./io.js";
|
|
2
2
|
import { type EngineOptions } from "../client/engine.js";
|
|
3
3
|
/** commander value-parser: a non-negative integer. */
|
|
4
4
|
export declare function parseIntArg(value: string): number;
|
|
@@ -66,6 +66,14 @@ export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
|
|
|
66
66
|
export declare function escapeControlChars(json: string): string;
|
|
67
67
|
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
68
68
|
export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
|
|
69
|
+
/**
|
|
70
|
+
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
71
|
+
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
72
|
+
* (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
|
|
73
|
+
* Called once per run, after the options are parsed and before the first request;
|
|
74
|
+
* stdout and the exit code are untouched.
|
|
75
|
+
*/
|
|
76
|
+
export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
|
|
69
77
|
export interface ActionContext {
|
|
70
78
|
client: ReturnType<CliDeps["createClient"]>;
|
|
71
79
|
global: GlobalOptions;
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Shared helpers used across CLI command groups: option parsers, the global
|
|
2
2
|
// option resolver, and the JSON result renderer.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { logOf } from "./io.js";
|
|
5
|
+
import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "../client/engine.js";
|
|
6
|
+
import { CkanError, cutForMessage, redactUrl } from "../client/errors.js";
|
|
6
7
|
import { findPortal } from "../client/portals.js";
|
|
7
8
|
import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
|
|
8
9
|
import { PORTALS } from "../client/portals-list.js";
|
|
@@ -104,7 +105,7 @@ export function parseBaseUrl(value) {
|
|
|
104
105
|
export function parsePortal(value) {
|
|
105
106
|
const portal = findPortal(value, PORTALS);
|
|
106
107
|
if (!portal) {
|
|
107
|
-
throw new InvalidArgumentError(`Unknown portal "${value}". \`ckan portals\` lists the known ones.`);
|
|
108
|
+
throw new InvalidArgumentError(`Unknown portal "${cutForMessage(redactUrl(value))}". \`ckan portals\` lists the known ones.`);
|
|
108
109
|
}
|
|
109
110
|
return portal.url;
|
|
110
111
|
}
|
|
@@ -169,6 +170,18 @@ export function renderJson(deps, global, value) {
|
|
|
169
170
|
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
170
171
|
deps.io.out(text);
|
|
171
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
175
|
+
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
176
|
+
* (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
|
|
177
|
+
* Called once per run, after the options are parsed and before the first request;
|
|
178
|
+
* stdout and the exit code are untouched.
|
|
179
|
+
*/
|
|
180
|
+
export function warnOnCleartext(deps, global) {
|
|
181
|
+
const problem = cleartextProblem(global.portal ?? global.baseUrl ?? DEFAULT_BASE_URL);
|
|
182
|
+
if (problem !== undefined)
|
|
183
|
+
logOf(deps).warn("http", problem);
|
|
184
|
+
}
|
|
172
185
|
/**
|
|
173
186
|
* Wrap an async command action with consistent global-option resolution and
|
|
174
187
|
* client construction. The callback receives a context (client + resolved global
|
|
@@ -182,6 +195,7 @@ export function action(deps, fn) {
|
|
|
182
195
|
const command = args[args.length - 1];
|
|
183
196
|
const positionals = args.slice(0, Math.max(0, args.length - 2));
|
|
184
197
|
const global = command.optsWithGlobals();
|
|
198
|
+
warnOnCleartext(deps, global);
|
|
185
199
|
const client = deps.createClient(toEngineOptions(global));
|
|
186
200
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
187
201
|
};
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
3
|
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
-
import {
|
|
4
|
+
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
5
|
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
7
|
/**
|
|
@@ -159,12 +159,13 @@ export class CkanClient {
|
|
|
159
159
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
160
160
|
// envelope; say so rather than "failed: undefined".
|
|
161
161
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
162
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
162
|
+
throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
163
163
|
}
|
|
164
164
|
if (!env.success) {
|
|
165
165
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
166
166
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
167
|
-
|
|
167
|
+
const type = env.error?.__type;
|
|
168
|
+
throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
|
|
168
169
|
}
|
|
169
170
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
170
171
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -116,6 +116,19 @@ export declare function sanitizeServerText(text: string): string;
|
|
|
116
116
|
* default transport still re-checks the scheme on every hop.
|
|
117
117
|
*/
|
|
118
118
|
export declare function validateBaseUrl(raw: string): string;
|
|
119
|
+
/**
|
|
120
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
121
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
122
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
123
|
+
* where nothing leaves the machine.
|
|
124
|
+
*
|
|
125
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
126
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
127
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
128
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
129
|
+
* (once per run, before the first request).
|
|
130
|
+
*/
|
|
131
|
+
export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
|
|
119
132
|
export declare class RequestEngine {
|
|
120
133
|
#private;
|
|
121
134
|
private readonly transport;
|
|
@@ -128,7 +141,8 @@ export declare class RequestEngine {
|
|
|
128
141
|
private readonly sleep;
|
|
129
142
|
constructor(options?: EngineOptions);
|
|
130
143
|
/**
|
|
131
|
-
* `text` without the base URL's credentials (raw and percent-decoded
|
|
144
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
145
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
132
146
|
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
133
147
|
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
134
148
|
* messages built from server text.
|