@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +55 -12
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -5
- package/dist/src/cli/io.js +65 -9
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +88 -11
- package/dist/src/cli/run.d.ts +53 -1
- package/dist/src/cli/run.js +239 -8
- package/dist/src/cli/shared.d.ts +17 -1
- package/dist/src/cli/shared.js +33 -3
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +124 -22
- package/dist/src/client/engine.d.ts +62 -9
- package/dist/src/client/engine.js +350 -47
- package/dist/src/client/errors.d.ts +81 -1
- package/dist/src/client/errors.js +171 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +3 -3
- package/dist/src/client/index.js +3 -3
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +39 -4
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +49 -3
- package/package.json +6 -5
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
// The CLI's log: every diagnostic line on stderr — errors, warnings, notes — is one
|
|
2
|
+
// record with a timestamp, a level and a topic. Two formats, chosen with the global
|
|
3
|
+
// `--log-format`:
|
|
4
|
+
//
|
|
5
|
+
// text 2026-10-09T14:03:12.481Z WARN [ckan.http] requests to … are sent unencrypted …
|
|
6
|
+
// jsonl {"ts":"2026-10-09T14:03:12.481Z","level":"WARN","topic":"ckan.http","msg":"…"}
|
|
7
|
+
//
|
|
8
|
+
// The text form follows log4j's pattern (`%d %-5p [%c] %m`), with the time in UTC
|
|
9
|
+
// ISO 8601. The topic is a dotted logger name: the program, then the area the record
|
|
10
|
+
// comes from (`ckan.cli`, `ckan.api`, `ckan.http`). stdout carries data
|
|
11
|
+
// only and is not touched; neither is `--help`/`--version`.
|
|
12
|
+
import { cutText, toWellFormed } from "../client/errors.js";
|
|
13
|
+
/** The log formats `--log-format` takes. */
|
|
14
|
+
export const LOG_FORMATS = ["text", "jsonl"];
|
|
15
|
+
/** The format without `--log-format`. */
|
|
16
|
+
export const DEFAULT_LOG_FORMAT = "text";
|
|
17
|
+
/** The program's name in every topic: `ckan.<area>`. */
|
|
18
|
+
export const LOG_PROGRAM = "ckan";
|
|
19
|
+
/** True for a character a record never carries raw (see `escapeForRecord`). */
|
|
20
|
+
function escapedInRecords(c) {
|
|
21
|
+
return ((c < 0x20 && c !== 0x09) || // C0 but TAB
|
|
22
|
+
(c >= 0x7f && c <= 0x9f) || // DEL and C1 (NEL, the 8-bit CSI)
|
|
23
|
+
c === 0x2028 || c === 0x2029 || // line and paragraph separator
|
|
24
|
+
c === 0x061c || c === 0x200e || c === 0x200f || // bidi marks
|
|
25
|
+
(c >= 0x202a && c <= 0x202e) || // bidi embeddings and overrides
|
|
26
|
+
(c >= 0x2066 && c <= 0x2069) // bidi isolates
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* `text` with every character that could split a record, forge a second one or steer
|
|
31
|
+
* the terminal written as an escape: CR as `\r`, LF as `\n`, any other C0 control but
|
|
32
|
+
* TAB, DEL and C1 as `\u00XX`, U+2028, U+2029 and the bidi controls (U+061C, U+200E,
|
|
33
|
+
* U+200F, U+202A–U+202E, U+2066–U+2069) as `\uXXXX`. Backslashes stay as they are. On
|
|
34
|
+
* the output of `JSON.stringify` (no raw C0 left) every escape it adds is valid JSON,
|
|
35
|
+
* so the same helper serves both formats. Checked by char code, so the source stays
|
|
36
|
+
* free of those characters.
|
|
37
|
+
*/
|
|
38
|
+
export function escapeForRecord(text) {
|
|
39
|
+
let out = "";
|
|
40
|
+
let from = 0;
|
|
41
|
+
for (let i = 0; i < text.length; i++) {
|
|
42
|
+
const c = text.charCodeAt(i);
|
|
43
|
+
if (!escapedInRecords(c))
|
|
44
|
+
continue;
|
|
45
|
+
const escaped = c === 0x0d ? "\\r" : c === 0x0a ? "\\n" : "\\u" + c.toString(16).padStart(4, "0");
|
|
46
|
+
out += text.slice(from, i) + escaped;
|
|
47
|
+
from = i + 1;
|
|
48
|
+
}
|
|
49
|
+
return from === 0 ? text : out + text.slice(from);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The longest message (in characters) a record carries. A longer one is cut at a
|
|
53
|
+
* code-point boundary and ends in `… (N more characters)`: one huge server text or typed
|
|
54
|
+
* value cannot flood stderr or a log store with a line of megabytes. Own messages that
|
|
55
|
+
* quote server or user data are bounded at their source too (`cutForMessage`); this
|
|
56
|
+
* is the backstop for every path.
|
|
57
|
+
*/
|
|
58
|
+
export const MAX_RECORD_MESSAGE = 4000;
|
|
59
|
+
/** `msg` within `MAX_RECORD_MESSAGE`, the cut marked with the number of characters left out. */
|
|
60
|
+
function boundRecordMessage(msg) {
|
|
61
|
+
if (msg.length <= MAX_RECORD_MESSAGE)
|
|
62
|
+
return msg;
|
|
63
|
+
const kept = cutText(msg, MAX_RECORD_MESSAGE);
|
|
64
|
+
return `${kept}… (${codePoints(msg, kept.length)} more characters)`;
|
|
65
|
+
}
|
|
66
|
+
/** The number of characters (code points) in `text` from `from` on. */
|
|
67
|
+
function codePoints(text, from) {
|
|
68
|
+
let n = 0;
|
|
69
|
+
for (let i = from; i < text.length; i++) {
|
|
70
|
+
const c = text.charCodeAt(i);
|
|
71
|
+
// The low half of a pair is not a character of its own.
|
|
72
|
+
if (c >= 0xdc00 && c <= 0xdfff && i > from && text.charCodeAt(i - 1) >= 0xd800 && text.charCodeAt(i - 1) <= 0xdbff)
|
|
73
|
+
continue;
|
|
74
|
+
n++;
|
|
75
|
+
}
|
|
76
|
+
return n;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* One record as one line, whatever the message holds: `escapeForRecord` runs over the
|
|
80
|
+
* message (text) or over the whole JSON object (jsonl), so no text that reaches a
|
|
81
|
+
* record can split it, forge another one, or reach the terminal as a control sequence.
|
|
82
|
+
*/
|
|
83
|
+
export function formatLogRecord(record, format) {
|
|
84
|
+
// Well-formed first: half a character would be `\ud83d` in jsonl, which jq rejects,
|
|
85
|
+
// stopping the whole stream.
|
|
86
|
+
const msg = toWellFormed(boundRecordMessage(record.msg));
|
|
87
|
+
if (format === "jsonl")
|
|
88
|
+
return escapeForRecord(JSON.stringify({ ts: record.ts, level: record.level, topic: record.topic, msg }));
|
|
89
|
+
return `${record.ts} ${record.level.padEnd(5)} [${record.topic}] ${escapeForRecord(msg)}`;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* A logger that writes each record, formatted, to `write` (stderr). `redact` replaces the
|
|
93
|
+
* secrets of the run in the message, before it is cut and escaped: the frame (time,
|
|
94
|
+
* level, topic) is never touched, so a secret equal to a year or a topic cannot corrupt
|
|
95
|
+
* it, and a secret holding DEL, C1 or bidi characters is found in its raw form.
|
|
96
|
+
*/
|
|
97
|
+
export function createLogger(options) {
|
|
98
|
+
const now = options.now ?? (() => new Date());
|
|
99
|
+
const redact = options.redact ?? ((text) => text);
|
|
100
|
+
const logger = {
|
|
101
|
+
format: options.format,
|
|
102
|
+
error: (area, msg) => log("ERROR", area, msg),
|
|
103
|
+
warn: (area, msg) => log("WARN", area, msg),
|
|
104
|
+
info: (area, msg) => log("INFO", area, msg),
|
|
105
|
+
};
|
|
106
|
+
const log = (level, area, msg) => options.write(formatLogRecord({ ts: now().toISOString(), level, topic: `${LOG_PROGRAM}.${area}`, msg: redact(msg) }, logger.format));
|
|
107
|
+
return logger;
|
|
108
|
+
}
|
|
109
|
+
/** Why `value` is not a log format, or undefined. */
|
|
110
|
+
export function logFormatProblem(value) {
|
|
111
|
+
return LOG_FORMATS.includes(value) ? undefined : `Expected one of ${LOG_FORMATS.join(", ")}.`;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The `--log-format` in `argv`, read before commander parses it: commander's own
|
|
115
|
+
* usage errors are logged too, and they happen while parsing. A missing or unknown
|
|
116
|
+
* value gives the default here; commander then reports an unknown one. The first
|
|
117
|
+
* `--log-format` counts: commander keeps it and rejects a second one (`once`), so that
|
|
118
|
+
* error is logged in the first one's format. This scan is only for the records of a
|
|
119
|
+
* parse error: once commander has parsed argv, its value is the format (the program's
|
|
120
|
+
* first preAction hook), so `--user-agent --log-format=jsonl` (a User-Agent) logs text.
|
|
121
|
+
* `valueOptions` names the program's options that take a value (`--user-agent`): the
|
|
122
|
+
* token after one is its value, never an option, as commander reads it, so `--user-agent
|
|
123
|
+
* --log-format jsonl` agrees with commander in a parse error too. A subcommand's value
|
|
124
|
+
* option (`--rows`) does not count: commander takes the program's own options out of
|
|
125
|
+
* argv first, so `search --rows --log-format jsonl` is jsonl and a `--rows` without its
|
|
126
|
+
* value.
|
|
127
|
+
*/
|
|
128
|
+
export function logFormatFromArgv(argv, valueOptions = new Set()) {
|
|
129
|
+
let format;
|
|
130
|
+
for (let i = 0; i < argv.length; i++) {
|
|
131
|
+
const token = argv[i];
|
|
132
|
+
if (token === "--")
|
|
133
|
+
break;
|
|
134
|
+
if (token !== "--log-format" && valueOptions.has(token)) {
|
|
135
|
+
i++;
|
|
136
|
+
continue;
|
|
137
|
+
}
|
|
138
|
+
if (token === "--log-format")
|
|
139
|
+
format = argv[i + 1];
|
|
140
|
+
else if (token.startsWith("--log-format="))
|
|
141
|
+
format = token.slice("--log-format=".length);
|
|
142
|
+
else
|
|
143
|
+
continue;
|
|
144
|
+
break;
|
|
145
|
+
}
|
|
146
|
+
return format !== undefined && logFormatProblem(format) === undefined ? format : DEFAULT_LOG_FORMAT;
|
|
147
|
+
}
|
|
148
|
+
/**
|
|
149
|
+
* Node's own process warnings (`(node:PID) Warning: …`, e.g. for
|
|
150
|
+
* `NODE_TLS_REJECT_UNAUTHORIZED=0`) as WARN records of `ckan.cli`:
|
|
151
|
+
* `(node) <name>: <message>`, in the log's format and with its redaction. Node's default
|
|
152
|
+
* listener, which prints the plain line, is removed (checked on Node 22, 24 and 26: that
|
|
153
|
+
* silences it; `--no-warnings` is not needed). The bin shim installs it once, before
|
|
154
|
+
* `run()`; a warning Node emits while it starts up, before any code runs, stays Node's.
|
|
155
|
+
*/
|
|
156
|
+
export function installWarningLog(source, log) {
|
|
157
|
+
source.removeAllListeners("warning");
|
|
158
|
+
source.on("warning", (warning) => log.warn("cli", `(node) ${warning.name}: ${warning.message}`));
|
|
159
|
+
}
|
|
@@ -3,4 +3,11 @@ import type { CliDeps } from "./io.js";
|
|
|
3
3
|
export declare const VERSION: string;
|
|
4
4
|
/** Default dependencies: real client + real stdout/stderr + real environment. */
|
|
5
5
|
export declare const defaultDeps: CliDeps;
|
|
6
|
+
/**
|
|
7
|
+
* commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
|
|
8
|
+
* or "" when nothing is close: the same rule as commander's own (an optimal-string-
|
|
9
|
+
* alignment distance below 3 that keeps more than 40 % of the word). The root action
|
|
10
|
+
* reports an unknown command itself, so commander's suggestion is rebuilt here.
|
|
11
|
+
*/
|
|
12
|
+
export declare function suggestSimilar(word: string, candidates: readonly string[]): string;
|
|
6
13
|
export declare function buildProgram(deps?: CliDeps): Command;
|
package/dist/src/cli/program.js
CHANGED
|
@@ -8,10 +8,11 @@ import { defaultIO } from "./io.js";
|
|
|
8
8
|
import { CkanClient } from "../client/client.js";
|
|
9
9
|
import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
|
|
10
10
|
import { MAX_TIMEOUT_MS } from "../client/http.js";
|
|
11
|
-
import { redactUrl } from "../client/errors.js";
|
|
12
|
-
import { parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
|
|
11
|
+
import { cutForMessage, redactUrl } from "../client/errors.js";
|
|
12
|
+
import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
|
|
13
13
|
import { registerCatalogueCommands } from "./commands/catalogue.js";
|
|
14
14
|
import { registerPortalCommands } from "./commands/portal.js";
|
|
15
|
+
import { DEFAULT_LOG_FORMAT, logFormatProblem } from "./log.js";
|
|
15
16
|
/**
|
|
16
17
|
* Single source of truth for the version: read from package.json at runtime
|
|
17
18
|
* rather than duplicating a literal that can silently drift after a release bump.
|
|
@@ -35,6 +36,60 @@ export const defaultDeps = {
|
|
|
35
36
|
env: process.env,
|
|
36
37
|
createClient: (options) => new CkanClient(options),
|
|
37
38
|
};
|
|
39
|
+
/** commander value-parser for `--log-format`. */
|
|
40
|
+
function parseLogFormat(value) {
|
|
41
|
+
const problem = logFormatProblem(value);
|
|
42
|
+
if (problem !== undefined)
|
|
43
|
+
throw new InvalidArgumentError(problem);
|
|
44
|
+
return value;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
|
|
48
|
+
* or "" when nothing is close: the same rule as commander's own (an optimal-string-
|
|
49
|
+
* alignment distance below 3 that keeps more than 40 % of the word). The root action
|
|
50
|
+
* reports an unknown command itself, so commander's suggestion is rebuilt here.
|
|
51
|
+
*/
|
|
52
|
+
export function suggestSimilar(word, candidates) {
|
|
53
|
+
let similar = [];
|
|
54
|
+
let best = 3;
|
|
55
|
+
for (const candidate of new Set(candidates)) {
|
|
56
|
+
if (candidate.length <= 1)
|
|
57
|
+
continue;
|
|
58
|
+
const distance = editDistance(word, candidate, best);
|
|
59
|
+
const length = Math.max(word.length, candidate.length);
|
|
60
|
+
if ((length - distance) / length <= 0.4)
|
|
61
|
+
continue;
|
|
62
|
+
if (distance < best) {
|
|
63
|
+
best = distance;
|
|
64
|
+
similar = [candidate];
|
|
65
|
+
}
|
|
66
|
+
else if (distance === best) {
|
|
67
|
+
similar.push(candidate);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
similar.sort((a, b) => a.localeCompare(b));
|
|
71
|
+
if (similar.length > 1)
|
|
72
|
+
return `\n(Did you mean one of ${similar.join(", ")}?)`;
|
|
73
|
+
return similar.length === 1 ? `\n(Did you mean ${similar[0]}?)` : "";
|
|
74
|
+
}
|
|
75
|
+
/** Optimal-string-alignment distance between `a` and `b`; at least `cap` when it is that far. */
|
|
76
|
+
function editDistance(a, b, cap) {
|
|
77
|
+
if (Math.abs(a.length - b.length) >= cap)
|
|
78
|
+
return Math.max(a.length, b.length);
|
|
79
|
+
const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
|
|
80
|
+
for (let j = 1; j <= b.length; j++)
|
|
81
|
+
d[0][j] = j;
|
|
82
|
+
for (let i = 1; i <= a.length; i++) {
|
|
83
|
+
for (let j = 1; j <= b.length; j++) {
|
|
84
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
85
|
+
let v = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
|
|
86
|
+
if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1])
|
|
87
|
+
v = Math.min(v, d[i - 2][j - 2] + 1);
|
|
88
|
+
d[i][j] = v;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return d[a.length][b.length];
|
|
92
|
+
}
|
|
38
93
|
export function buildProgram(deps = defaultDeps) {
|
|
39
94
|
const program = new Command();
|
|
40
95
|
const baseUrlDefault = deps.env["CKAN_BASE_URL"] || DEFAULT_BASE_URL;
|
|
@@ -43,17 +98,18 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
43
98
|
.description("CLI for any CKAN open-data portal (Action API v3)")
|
|
44
99
|
.version(VERSION)
|
|
45
100
|
.addOption(new Option("--base-url <url>", "CKAN site URL (env CKAN_BASE_URL)")
|
|
46
|
-
.argParser(parseBaseUrl)
|
|
101
|
+
.argParser(once(parseBaseUrl))
|
|
47
102
|
// The help shows the default without userinfo: a password in
|
|
48
103
|
// CKAN_BASE_URL must not end up in `--help` output or CI logs.
|
|
49
104
|
.default(baseUrlDefault, JSON.stringify(redactUrl(baseUrlDefault))))
|
|
50
105
|
.addOption(new Option("--portal <id>", "a known portal by id (see `ckan portals`)")
|
|
51
|
-
.argParser(parsePortal)
|
|
106
|
+
.argParser(once(parsePortal))
|
|
52
107
|
.conflicts("baseUrl"))
|
|
53
|
-
.option("--timeout <ms>", "per-request timeout in milliseconds", parseBoundedInt(0, MAX_TIMEOUT_MS))
|
|
54
|
-
.option("--user-agent <ua>", "User-Agent header value", parseHeaderValue)
|
|
55
|
-
.option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each waits
|
|
56
|
-
.option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", parseIntArg)
|
|
108
|
+
.option("--timeout <ms>", "per-request timeout in milliseconds", once(parseBoundedInt(0, MAX_TIMEOUT_MS)))
|
|
109
|
+
.option("--user-agent <ua>", "User-Agent header value", once(parseHeaderValue))
|
|
110
|
+
.option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each backs off linearly, or waits a longer Retry-After up to ${MAX_RETRY_AFTER_MS / 1000} s)`, once(parseBoundedInt(0, MAX_RETRIES)))
|
|
111
|
+
.option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", once(parseIntArg))
|
|
112
|
+
.option("--log-format <format>", `how errors, warnings and notes are written to stderr: text (log4j style: time, level, [topic], message) or jsonl (one JSON object per line: ts, level, topic, msg); default ${DEFAULT_LOG_FORMAT}`, once(parseLogFormat))
|
|
57
113
|
.option("--compact", "print JSON on a single line instead of pretty-printed")
|
|
58
114
|
.showHelpAfterError();
|
|
59
115
|
// A bare invocation (no subcommand) should print help to stdout and exit 0,
|
|
@@ -68,13 +124,34 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
68
124
|
program.action(() => {
|
|
69
125
|
const [unknown] = program.args;
|
|
70
126
|
if (unknown !== undefined) {
|
|
71
|
-
|
|
127
|
+
// The name is the user's: cut, after its credentials are redacted (a cut could
|
|
128
|
+
// otherwise leave part of a password without the "@" the redaction keys on).
|
|
129
|
+
const names = program.commands.map((c) => c.name()).filter((name) => name !== "help");
|
|
130
|
+
program.error(`error: unknown command '${cutForMessage(redactUrl(unknown))}'${suggestSimilar(unknown, names)}`, {
|
|
131
|
+
code: "commander.unknownCommand",
|
|
132
|
+
});
|
|
72
133
|
}
|
|
73
134
|
program.help();
|
|
74
135
|
});
|
|
75
136
|
// commander runs value parsers on flags but not on defaults, so a base URL
|
|
76
|
-
// taken from CKAN_BASE_URL is checked here, before
|
|
77
|
-
|
|
137
|
+
// taken from CKAN_BASE_URL is checked here, before a command that uses it runs.
|
|
138
|
+
// The commands that never use it skip the check: help (the root action prints
|
|
139
|
+
// help or names an unknown command, `help` prints help) and `portals`, which
|
|
140
|
+
// prints the built-in list or checks each portal at its own URL — a user whose
|
|
141
|
+
// variable is broken is exactly the one looking for a working portal id.
|
|
142
|
+
const offline = new Set(["help", "portals"]);
|
|
143
|
+
// One source for the log format once commander has parsed argv: its value, not the
|
|
144
|
+
// scan of argv (an option's value can look like --log-format; `--` ends the scan, not
|
|
145
|
+
// commander's parse of a value). The first hook, so the CKAN_BASE_URL check below
|
|
146
|
+
// already logs in that format.
|
|
147
|
+
program.hook("preAction", (_program, actionCommand) => {
|
|
148
|
+
const format = actionCommand.optsWithGlobals().logFormat;
|
|
149
|
+
if (deps.log !== undefined)
|
|
150
|
+
deps.log.format = format ?? DEFAULT_LOG_FORMAT;
|
|
151
|
+
});
|
|
152
|
+
program.hook("preAction", (_program, actionCommand) => {
|
|
153
|
+
if (actionCommand === program || offline.has(actionCommand.name()))
|
|
154
|
+
return;
|
|
78
155
|
if (program.getOptionValueSource("baseUrl") !== "default" || program.opts()["portal"] !== undefined)
|
|
79
156
|
return;
|
|
80
157
|
try {
|
package/dist/src/cli/run.d.ts
CHANGED
|
@@ -1,2 +1,54 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type Command } from "commander";
|
|
2
|
+
import { type CliDeps } from "./io.js";
|
|
3
|
+
import { type Logger } from "./log.js";
|
|
4
|
+
/**
|
|
5
|
+
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
6
|
+
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
7
|
+
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
8
|
+
* `withRedactedOutput`.
|
|
9
|
+
*/
|
|
10
|
+
export declare function redactUserinfo(text: string): string;
|
|
11
|
+
/** The secrets of a run, and the two ways they are replaced. */
|
|
12
|
+
export interface Redaction {
|
|
13
|
+
/**
|
|
14
|
+
* stdout text: the userinfo of every URL-like argument and of CKAN_BASE_URL replaced
|
|
15
|
+
* (`***@`), and the forms a server echoes it back in (the Basic value, the decoded
|
|
16
|
+
* `user:password`).
|
|
17
|
+
*/
|
|
18
|
+
out(text: string): string;
|
|
19
|
+
/** stderr text, a record's message: that, and the password alone (`***`). */
|
|
20
|
+
err(text: string): string;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
|
|
24
|
+
* its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
|
|
25
|
+
* own messages name the values they refuse, and help shows the base URL's default:
|
|
26
|
+
* whatever path a credential takes to stdout or stderr, the exact userinfo (as
|
|
27
|
+
* `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
|
|
28
|
+
* by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
|
|
29
|
+
* `/`; the exact strings can. Without credentials the text passes through unchanged.
|
|
30
|
+
*/
|
|
31
|
+
export declare function redactionFor(argv: readonly string[], env: Record<string, string | undefined>): Redaction;
|
|
32
|
+
/**
|
|
33
|
+
* `deps` that keep the secrets of this run (`redactionFor`) out of everything they
|
|
34
|
+
* print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
|
|
35
|
+
* message before formatting it, then writes to the raw `io.err`, so the frame is never
|
|
36
|
+
* touched and a password with DEL, C1 or bidi characters is matched in its raw form.
|
|
37
|
+
* `io.err` itself is redacted too, for anything that writes to stderr without the log.
|
|
38
|
+
*/
|
|
39
|
+
export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[]): CliDeps;
|
|
40
|
+
/**
|
|
41
|
+
* The index of a `help` command whose topic names no command (`ckan help nonexistent`),
|
|
42
|
+
* or undefined. Commander answers that with the root help on stderr and exit 1, but
|
|
43
|
+
* no line saying what was wrong. Global options before `help` are skipped with their
|
|
44
|
+
* values; scanning stops at `--`.
|
|
45
|
+
*/
|
|
46
|
+
export declare function unknownHelpTopic(program: Command, argv: readonly string[]): number | undefined;
|
|
47
|
+
/**
|
|
48
|
+
* The log for what happens outside `run()`, in the bin shim: a stdout write error
|
|
49
|
+
* (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
|
|
50
|
+
* for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
|
|
51
|
+
* run's own log; it writes to the raw stderr.
|
|
52
|
+
*/
|
|
53
|
+
export declare function processLogger(argv: readonly string[], env?: Record<string, string | undefined>): Logger;
|
|
2
54
|
export declare function run(argv: string[], deps?: CliDeps): Promise<number>;
|
package/dist/src/cli/run.js
CHANGED
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
// captured output and exit code without spawning a subprocess.
|
|
4
4
|
import { CommanderError } from "commander";
|
|
5
5
|
import { buildProgram, defaultDeps } from "./program.js";
|
|
6
|
-
import {
|
|
6
|
+
import { logOf } from "./io.js";
|
|
7
|
+
import { createLogger, logFormatFromArgv } from "./log.js";
|
|
8
|
+
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "../client/errors.js";
|
|
7
9
|
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
8
10
|
const USAGE_EXIT = 1;
|
|
9
11
|
/**
|
|
@@ -11,18 +13,241 @@ const USAGE_EXIT = 1;
|
|
|
11
13
|
* commander does not propagate these to subcommands, so a parse error on a
|
|
12
14
|
* subcommand would otherwise call process.exit() and bypass our error handling.
|
|
13
15
|
*/
|
|
14
|
-
function configureTree(command, deps) {
|
|
16
|
+
function configureTree(command, deps, state = { errorLogged: false }) {
|
|
15
17
|
command.exitOverride();
|
|
16
18
|
command.configureOutput({
|
|
17
19
|
writeOut: (str) => deps.io.out(str.replace(/\n$/, "")),
|
|
18
|
-
writeErr: (str) => deps
|
|
20
|
+
writeErr: (str) => writeCommanderErr(command, deps, state, str),
|
|
19
21
|
});
|
|
20
22
|
for (const child of command.commands)
|
|
21
|
-
configureTree(child, deps);
|
|
23
|
+
configureTree(child, deps, state);
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* The names (long and short) of the program's own options that require a value
|
|
27
|
+
* (`--user-agent`). Only the program's: commander takes them out of argv wherever they
|
|
28
|
+
* stand, before a subcommand sees the rest, so a subcommand's `--rows` never swallows
|
|
29
|
+
* a `--log-format` after it.
|
|
30
|
+
*/
|
|
31
|
+
function valueOptionsOf(program) {
|
|
32
|
+
const names = new Set();
|
|
33
|
+
for (const option of program.options) {
|
|
34
|
+
if (!option.required)
|
|
35
|
+
continue;
|
|
36
|
+
if (option.long !== undefined)
|
|
37
|
+
names.add(option.long);
|
|
38
|
+
if (option.short !== undefined)
|
|
39
|
+
names.add(option.short);
|
|
40
|
+
}
|
|
41
|
+
return names;
|
|
42
|
+
}
|
|
43
|
+
/** `ckan search`: the command's name with its parents'. */
|
|
44
|
+
function commandPath(command) {
|
|
45
|
+
const names = [];
|
|
46
|
+
for (let c = command; c !== null; c = c.parent)
|
|
47
|
+
names.unshift(c.name());
|
|
48
|
+
return names.join(" ");
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* commander's stderr output as log records, one per line. Its `error: …` is an ERROR of
|
|
52
|
+
* `cli`, with a following `(Did you mean …?)` line appended to that same record; the
|
|
53
|
+
* help it shows after an error is one INFO record per non-blank line. A command group
|
|
54
|
+
* run without its subcommand makes commander show the help as an error with no
|
|
55
|
+
* `error:` line: an ERROR record "missing command: `ckan <group> <subcommand>`" comes
|
|
56
|
+
* first, so every failed run has one (ckan has no groups today; a bare `ckan` prints
|
|
57
|
+
* its help to stdout and exits 0).
|
|
58
|
+
*/
|
|
59
|
+
function writeCommanderErr(command, deps, state, str) {
|
|
60
|
+
const log = logOf(deps);
|
|
61
|
+
const text = str.replace(/\n$/, "");
|
|
62
|
+
// The blank line commander writes between an error and the help it shows after.
|
|
63
|
+
if (text.trim() === "")
|
|
64
|
+
return;
|
|
65
|
+
if (text.startsWith("error: ")) {
|
|
66
|
+
state.errorLogged = true;
|
|
67
|
+
log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
if (!state.errorLogged) {
|
|
71
|
+
state.errorLogged = true;
|
|
72
|
+
log.error("cli", `missing command: \`${commandPath(command)} <subcommand>\``);
|
|
73
|
+
}
|
|
74
|
+
for (const line of text.split("\n"))
|
|
75
|
+
if (line.trim() !== "")
|
|
76
|
+
log.info("cli", line.trimEnd());
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
80
|
+
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
81
|
+
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
82
|
+
* `withRedactedOutput`.
|
|
83
|
+
*/
|
|
84
|
+
export function redactUserinfo(text) {
|
|
85
|
+
return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* The options whose value is a base URL (`--portal` takes a portal's URL as well as its
|
|
89
|
+
* id): a `user:password@host` given there without its scheme is still a credential, like
|
|
90
|
+
* one in CKAN_BASE_URL (anywhere else a bare `a:b@c` is not).
|
|
91
|
+
*/
|
|
92
|
+
const BASE_URL_FLAGS = ["--base-url", "--portal"];
|
|
93
|
+
/** The values of the `flags` in `argv`, in both forms (`--flag value`, `--flag=value`). */
|
|
94
|
+
function flagValues(argv, flags) {
|
|
95
|
+
const found = [];
|
|
96
|
+
argv.forEach((token, i) => {
|
|
97
|
+
const next = argv[i + 1];
|
|
98
|
+
if (flags.includes(token) && next !== undefined)
|
|
99
|
+
found.push(next);
|
|
100
|
+
const eq = token.indexOf("=");
|
|
101
|
+
if (eq > 0 && flags.includes(token.slice(0, eq)))
|
|
102
|
+
found.push(token.slice(eq + 1));
|
|
103
|
+
});
|
|
104
|
+
return found;
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
|
|
108
|
+
* its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
|
|
109
|
+
* own messages name the values they refuse, and help shows the base URL's default:
|
|
110
|
+
* whatever path a credential takes to stdout or stderr, the exact userinfo (as
|
|
111
|
+
* `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
|
|
112
|
+
* by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
|
|
113
|
+
* `/`; the exact strings can. Without credentials the text passes through unchanged.
|
|
114
|
+
*/
|
|
115
|
+
export function redactionFor(argv, env) {
|
|
116
|
+
// An `--option=value` token is echoed as its value alone.
|
|
117
|
+
const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
|
|
118
|
+
// A base URL typed without its scheme is read as if it had one (anywhere else a bare
|
|
119
|
+
// `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
|
|
120
|
+
const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
|
|
121
|
+
const sources = [...values, ...baseUrls];
|
|
122
|
+
const secrets = new Set();
|
|
123
|
+
const encoded = new Set();
|
|
124
|
+
const echoed = new Set();
|
|
125
|
+
const passwords = new Set();
|
|
126
|
+
for (const source of sources) {
|
|
127
|
+
for (const secret of credentialsIn(source)) {
|
|
128
|
+
secrets.add(secret);
|
|
129
|
+
secrets.add(JSON.stringify(secret).slice(1, -1));
|
|
130
|
+
// A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
|
|
131
|
+
// 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
|
|
132
|
+
encoded.add(encodeURIComponent(`${secret}@`));
|
|
133
|
+
// What a server echoes back: the Basic value and the decoded user:password on
|
|
134
|
+
// stdout and stderr, the password alone (it may well occur in the data) on stderr.
|
|
135
|
+
const [basic, pair, password] = echoedCredentialForms(secret);
|
|
136
|
+
if (basic !== undefined)
|
|
137
|
+
echoed.add(basic);
|
|
138
|
+
if (pair !== undefined)
|
|
139
|
+
echoed.add(pair);
|
|
140
|
+
if (password !== undefined)
|
|
141
|
+
passwords.add(password);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
if (secrets.size === 0)
|
|
145
|
+
return { out: (text) => text, err: (text) => text };
|
|
146
|
+
const list = [...secrets];
|
|
147
|
+
// Longest first, so a secret is never left half-replaced by one of its own substrings.
|
|
148
|
+
const echoedList = [...echoed].sort((a, b) => b.length - a.length);
|
|
149
|
+
const passwordList = [...passwords].sort((a, b) => b.length - a.length);
|
|
150
|
+
const out = (text) => {
|
|
151
|
+
let result = redactUserinfo(redactCredentials(text, list));
|
|
152
|
+
for (const form of encoded)
|
|
153
|
+
result = result.split(form).join("***%40");
|
|
154
|
+
return redactSecrets(result, echoedList);
|
|
155
|
+
};
|
|
156
|
+
return { out, err: (text) => redactSecrets(out(text), passwordList) };
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* `deps` that keep the secrets of this run (`redactionFor`) out of everything they
|
|
160
|
+
* print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
|
|
161
|
+
* message before formatting it, then writes to the raw `io.err`, so the frame is never
|
|
162
|
+
* touched and a password with DEL, C1 or bidi characters is matched in its raw form.
|
|
163
|
+
* `io.err` itself is redacted too, for anything that writes to stderr without the log.
|
|
164
|
+
*/
|
|
165
|
+
export function withRedactedOutput(deps, argv) {
|
|
166
|
+
const redaction = redactionFor(argv, deps.env);
|
|
167
|
+
const { out, err } = deps.io;
|
|
168
|
+
return {
|
|
169
|
+
...deps,
|
|
170
|
+
io: { ...deps.io, out: (text) => out(redaction.out(text)), err: (text) => err(redaction.err(text)) },
|
|
171
|
+
log: createLogger({
|
|
172
|
+
format: logFormatFromArgv(argv),
|
|
173
|
+
write: err,
|
|
174
|
+
redact: redaction.err,
|
|
175
|
+
...(deps.now === undefined ? {} : { now: deps.now }),
|
|
176
|
+
}),
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* The index of a `help` command whose topic names no command (`ckan help nonexistent`),
|
|
181
|
+
* or undefined. Commander answers that with the root help on stderr and exit 1, but
|
|
182
|
+
* no line saying what was wrong. Global options before `help` are skipped with their
|
|
183
|
+
* values; scanning stops at `--`.
|
|
184
|
+
*/
|
|
185
|
+
export function unknownHelpTopic(program, argv) {
|
|
186
|
+
for (let i = 0; i < argv.length; i++) {
|
|
187
|
+
const token = argv[i];
|
|
188
|
+
if (token === "--")
|
|
189
|
+
return undefined;
|
|
190
|
+
if (token.startsWith("-")) {
|
|
191
|
+
if (token.includes("="))
|
|
192
|
+
continue;
|
|
193
|
+
const option = program.options.find((o) => o.long === token || o.short === token);
|
|
194
|
+
if (option?.required)
|
|
195
|
+
i++;
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
if (token !== "help")
|
|
199
|
+
return undefined;
|
|
200
|
+
const topic = argv[i + 1];
|
|
201
|
+
if (topic === undefined || topic.startsWith("-"))
|
|
202
|
+
return undefined;
|
|
203
|
+
const known = program.commands.some((c) => c.name() === topic || c.aliases().includes(topic));
|
|
204
|
+
return known ? undefined : i;
|
|
205
|
+
}
|
|
206
|
+
return undefined;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* The log for what happens outside `run()`, in the bin shim: a stdout write error
|
|
210
|
+
* (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
|
|
211
|
+
* for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
|
|
212
|
+
* run's own log; it writes to the raw stderr.
|
|
213
|
+
*/
|
|
214
|
+
export function processLogger(argv, env = process.env) {
|
|
215
|
+
return createLogger({
|
|
216
|
+
format: logFormatFromArgv(argv),
|
|
217
|
+
write: (line) => process.stderr.write(line + "\n"),
|
|
218
|
+
redact: redactionFor(argv, env).err,
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* The log area of a `CkanError` that is neither an API error nor a usage error: the
|
|
223
|
+
* connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
|
|
224
|
+
* envelope, the wrong result shape, an unknown charset — the portal's answer as much as
|
|
225
|
+
* an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
|
|
226
|
+
* envelope, `CkanActionError`), else `cli` (a response nested too deeply to print is
|
|
227
|
+
* about printing the answer, not its shape).
|
|
228
|
+
*/
|
|
229
|
+
function areaOf(err) {
|
|
230
|
+
if (err instanceof CkanNetworkError)
|
|
231
|
+
return "http";
|
|
232
|
+
if (err instanceof CkanParseError || err instanceof CkanActionError)
|
|
233
|
+
return "api";
|
|
234
|
+
return "cli";
|
|
22
235
|
}
|
|
23
236
|
export async function run(argv, deps = defaultDeps) {
|
|
237
|
+
// The log replaces the secrets of the run in every message, in either format.
|
|
238
|
+
deps = withRedactedOutput(deps, argv);
|
|
24
239
|
const program = buildProgram(deps);
|
|
25
240
|
configureTree(program, deps);
|
|
241
|
+
// For the records of a parse error: the scan of argv, now knowing which of the program's
|
|
242
|
+
// options take a value, as commander reads them. Once commander has parsed argv, the program's
|
|
243
|
+
// first preAction hook sets the format it parsed (`buildProgram`).
|
|
244
|
+
if (deps.log !== undefined)
|
|
245
|
+
deps.log.format = logFormatFromArgv(argv, valueOptionsOf(program));
|
|
246
|
+
// `help <unknown>` is answered like `<unknown>`: "error: unknown command '<unknown>'",
|
|
247
|
+
// the help after it, exit 1 — not the help alone with a failure code.
|
|
248
|
+
const helpAt = unknownHelpTopic(program, argv);
|
|
249
|
+
if (helpAt !== undefined)
|
|
250
|
+
argv = argv.filter((_, i) => i !== helpAt);
|
|
26
251
|
try {
|
|
27
252
|
await program.parseAsync(argv, { from: "user" });
|
|
28
253
|
return 0;
|
|
@@ -32,24 +257,30 @@ export async function run(argv, deps = defaultDeps) {
|
|
|
32
257
|
// Help/version requests exit 0; genuine parse errors carry their own code.
|
|
33
258
|
return err.exitCode;
|
|
34
259
|
}
|
|
260
|
+
const log = logOf(deps);
|
|
35
261
|
if (err instanceof CkanValidationError) {
|
|
36
262
|
// An input the library refused before any request (a rule the commander
|
|
37
263
|
// parsers do not see, e.g. one across parameters): a usage error.
|
|
38
|
-
|
|
264
|
+
log.error("cli", err.message);
|
|
39
265
|
return USAGE_EXIT;
|
|
40
266
|
}
|
|
41
267
|
if (err instanceof CkanApiError) {
|
|
42
|
-
|
|
268
|
+
log.error("api", err.message);
|
|
43
269
|
// Map a few notable statuses to distinct exit codes for scripting.
|
|
44
270
|
if (err.status === 404)
|
|
45
271
|
return 4;
|
|
46
272
|
return 1;
|
|
47
273
|
}
|
|
274
|
+
if (err instanceof CkanActionError) {
|
|
275
|
+
log.error(areaOf(err), err.message);
|
|
276
|
+
// CKAN's Not Found Error is a not-found whatever the HTTP status it came with.
|
|
277
|
+
return err.isNotFound ? 4 : 1;
|
|
278
|
+
}
|
|
48
279
|
if (err instanceof CkanError) {
|
|
49
|
-
|
|
280
|
+
log.error(areaOf(err), err.message);
|
|
50
281
|
return 1;
|
|
51
282
|
}
|
|
52
|
-
|
|
283
|
+
log.error("cli", `Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
|
|
53
284
|
return 1;
|
|
54
285
|
}
|
|
55
286
|
}
|