@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,159 @@
1
+ // The CLI's log: every diagnostic line on stderr — errors, warnings, notes — is one
2
+ // record with a timestamp, a level and a topic. Two formats, chosen with the global
3
+ // `--log-format`:
4
+ //
5
+ // text 2026-10-09T14:03:12.481Z WARN [ckan.http] requests to … are sent unencrypted …
6
+ // jsonl {"ts":"2026-10-09T14:03:12.481Z","level":"WARN","topic":"ckan.http","msg":"…"}
7
+ //
8
+ // The text form follows log4j's pattern (`%d %-5p [%c] %m`), with the time in UTC
9
+ // ISO 8601. The topic is a dotted logger name: the program, then the area the record
10
+ // comes from (`ckan.cli`, `ckan.api`, `ckan.http`). stdout carries data
11
+ // only and is not touched; neither is `--help`/`--version`.
12
+ import { cutText, toWellFormed } from "../client/errors.js";
13
+ /** The log formats `--log-format` takes. */
14
+ export const LOG_FORMATS = ["text", "jsonl"];
15
+ /** The format without `--log-format`. */
16
+ export const DEFAULT_LOG_FORMAT = "text";
17
+ /** The program's name in every topic: `ckan.<area>`. */
18
+ export const LOG_PROGRAM = "ckan";
19
+ /** True for a character a record never carries raw (see `escapeForRecord`). */
20
+ function escapedInRecords(c) {
21
+ return ((c < 0x20 && c !== 0x09) || // C0 but TAB
22
+ (c >= 0x7f && c <= 0x9f) || // DEL and C1 (NEL, the 8-bit CSI)
23
+ c === 0x2028 || c === 0x2029 || // line and paragraph separator
24
+ c === 0x061c || c === 0x200e || c === 0x200f || // bidi marks
25
+ (c >= 0x202a && c <= 0x202e) || // bidi embeddings and overrides
26
+ (c >= 0x2066 && c <= 0x2069) // bidi isolates
27
+ );
28
+ }
29
+ /**
30
+ * `text` with every character that could split a record, forge a second one or steer
31
+ * the terminal written as an escape: CR as `\r`, LF as `\n`, any other C0 control but
32
+ * TAB, DEL and C1 as `\u00XX`, U+2028, U+2029 and the bidi controls (U+061C, U+200E,
33
+ * U+200F, U+202A–U+202E, U+2066–U+2069) as `\uXXXX`. Backslashes stay as they are. On
34
+ * the output of `JSON.stringify` (no raw C0 left) every escape it adds is valid JSON,
35
+ * so the same helper serves both formats. Checked by char code, so the source stays
36
+ * free of those characters.
37
+ */
38
+ export function escapeForRecord(text) {
39
+ let out = "";
40
+ let from = 0;
41
+ for (let i = 0; i < text.length; i++) {
42
+ const c = text.charCodeAt(i);
43
+ if (!escapedInRecords(c))
44
+ continue;
45
+ const escaped = c === 0x0d ? "\\r" : c === 0x0a ? "\\n" : "\\u" + c.toString(16).padStart(4, "0");
46
+ out += text.slice(from, i) + escaped;
47
+ from = i + 1;
48
+ }
49
+ return from === 0 ? text : out + text.slice(from);
50
+ }
51
+ /**
52
+ * The longest message (in characters) a record carries. A longer one is cut at a
53
+ * code-point boundary and ends in `… (N more characters)`: one huge server text or typed
54
+ * value cannot flood stderr or a log store with a line of megabytes. Own messages that
55
+ * quote server or user data are bounded at their source too (`cutForMessage`); this
56
+ * is the backstop for every path.
57
+ */
58
+ export const MAX_RECORD_MESSAGE = 4000;
59
+ /** `msg` within `MAX_RECORD_MESSAGE`, the cut marked with the number of characters left out. */
60
+ function boundRecordMessage(msg) {
61
+ if (msg.length <= MAX_RECORD_MESSAGE)
62
+ return msg;
63
+ const kept = cutText(msg, MAX_RECORD_MESSAGE);
64
+ return `${kept}… (${codePoints(msg, kept.length)} more characters)`;
65
+ }
66
+ /** The number of characters (code points) in `text` from `from` on. */
67
+ function codePoints(text, from) {
68
+ let n = 0;
69
+ for (let i = from; i < text.length; i++) {
70
+ const c = text.charCodeAt(i);
71
+ // The low half of a pair is not a character of its own.
72
+ if (c >= 0xdc00 && c <= 0xdfff && i > from && text.charCodeAt(i - 1) >= 0xd800 && text.charCodeAt(i - 1) <= 0xdbff)
73
+ continue;
74
+ n++;
75
+ }
76
+ return n;
77
+ }
78
+ /**
79
+ * One record as one line, whatever the message holds: `escapeForRecord` runs over the
80
+ * message (text) or over the whole JSON object (jsonl), so no text that reaches a
81
+ * record can split it, forge another one, or reach the terminal as a control sequence.
82
+ */
83
+ export function formatLogRecord(record, format) {
84
+ // Well-formed first: half a character would be `\ud83d` in jsonl, which jq rejects,
85
+ // stopping the whole stream.
86
+ const msg = toWellFormed(boundRecordMessage(record.msg));
87
+ if (format === "jsonl")
88
+ return escapeForRecord(JSON.stringify({ ts: record.ts, level: record.level, topic: record.topic, msg }));
89
+ return `${record.ts} ${record.level.padEnd(5)} [${record.topic}] ${escapeForRecord(msg)}`;
90
+ }
91
+ /**
92
+ * A logger that writes each record, formatted, to `write` (stderr). `redact` replaces the
93
+ * secrets of the run in the message, before it is cut and escaped: the frame (time,
94
+ * level, topic) is never touched, so a secret equal to a year or a topic cannot corrupt
95
+ * it, and a secret holding DEL, C1 or bidi characters is found in its raw form.
96
+ */
97
+ export function createLogger(options) {
98
+ const now = options.now ?? (() => new Date());
99
+ const redact = options.redact ?? ((text) => text);
100
+ const logger = {
101
+ format: options.format,
102
+ error: (area, msg) => log("ERROR", area, msg),
103
+ warn: (area, msg) => log("WARN", area, msg),
104
+ info: (area, msg) => log("INFO", area, msg),
105
+ };
106
+ const log = (level, area, msg) => options.write(formatLogRecord({ ts: now().toISOString(), level, topic: `${LOG_PROGRAM}.${area}`, msg: redact(msg) }, logger.format));
107
+ return logger;
108
+ }
109
+ /** Why `value` is not a log format, or undefined. */
110
+ export function logFormatProblem(value) {
111
+ return LOG_FORMATS.includes(value) ? undefined : `Expected one of ${LOG_FORMATS.join(", ")}.`;
112
+ }
113
+ /**
114
+ * The `--log-format` in `argv`, read before commander parses it: commander's own
115
+ * usage errors are logged too, and they happen while parsing. A missing or unknown
116
+ * value gives the default here; commander then reports an unknown one. The first
117
+ * `--log-format` counts: commander keeps it and rejects a second one (`once`), so that
118
+ * error is logged in the first one's format. This scan is only for the records of a
119
+ * parse error: once commander has parsed argv, its value is the format (the program's
120
+ * first preAction hook), so `--user-agent --log-format=jsonl` (a User-Agent) logs text.
121
+ * `valueOptions` names the program's options that take a value (`--user-agent`): the
122
+ * token after one is its value, never an option, as commander reads it, so `--user-agent
123
+ * --log-format jsonl` agrees with commander in a parse error too. A subcommand's value
124
+ * option (`--rows`) does not count: commander takes the program's own options out of
125
+ * argv first, so `search --rows --log-format jsonl` is jsonl and a `--rows` without its
126
+ * value.
127
+ */
128
+ export function logFormatFromArgv(argv, valueOptions = new Set()) {
129
+ let format;
130
+ for (let i = 0; i < argv.length; i++) {
131
+ const token = argv[i];
132
+ if (token === "--")
133
+ break;
134
+ if (token !== "--log-format" && valueOptions.has(token)) {
135
+ i++;
136
+ continue;
137
+ }
138
+ if (token === "--log-format")
139
+ format = argv[i + 1];
140
+ else if (token.startsWith("--log-format="))
141
+ format = token.slice("--log-format=".length);
142
+ else
143
+ continue;
144
+ break;
145
+ }
146
+ return format !== undefined && logFormatProblem(format) === undefined ? format : DEFAULT_LOG_FORMAT;
147
+ }
148
+ /**
149
+ * Node's own process warnings (`(node:PID) Warning: …`, e.g. for
150
+ * `NODE_TLS_REJECT_UNAUTHORIZED=0`) as WARN records of `ckan.cli`:
151
+ * `(node) <name>: <message>`, in the log's format and with its redaction. Node's default
152
+ * listener, which prints the plain line, is removed (checked on Node 22, 24 and 26: that
153
+ * silences it; `--no-warnings` is not needed). The bin shim installs it once, before
154
+ * `run()`; a warning Node emits while it starts up, before any code runs, stays Node's.
155
+ */
156
+ export function installWarningLog(source, log) {
157
+ source.removeAllListeners("warning");
158
+ source.on("warning", (warning) => log.warn("cli", `(node) ${warning.name}: ${warning.message}`));
159
+ }
@@ -3,4 +3,11 @@ import type { CliDeps } from "./io.js";
3
3
  export declare const VERSION: string;
4
4
  /** Default dependencies: real client + real stdout/stderr + real environment. */
5
5
  export declare const defaultDeps: CliDeps;
6
+ /**
7
+ * commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
8
+ * or "" when nothing is close: the same rule as commander's own (an optimal-string-
9
+ * alignment distance below 3 that keeps more than 40 % of the word). The root action
10
+ * reports an unknown command itself, so commander's suggestion is rebuilt here.
11
+ */
12
+ export declare function suggestSimilar(word: string, candidates: readonly string[]): string;
6
13
  export declare function buildProgram(deps?: CliDeps): Command;
@@ -8,10 +8,11 @@ import { defaultIO } from "./io.js";
8
8
  import { CkanClient } from "../client/client.js";
9
9
  import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
10
10
  import { MAX_TIMEOUT_MS } from "../client/http.js";
11
- import { redactUrl } from "../client/errors.js";
12
- import { parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
11
+ import { cutForMessage, redactUrl } from "../client/errors.js";
12
+ import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
13
13
  import { registerCatalogueCommands } from "./commands/catalogue.js";
14
14
  import { registerPortalCommands } from "./commands/portal.js";
15
+ import { DEFAULT_LOG_FORMAT, logFormatProblem } from "./log.js";
15
16
  /**
16
17
  * Single source of truth for the version: read from package.json at runtime
17
18
  * rather than duplicating a literal that can silently drift after a release bump.
@@ -35,6 +36,60 @@ export const defaultDeps = {
35
36
  env: process.env,
36
37
  createClient: (options) => new CkanClient(options),
37
38
  };
39
+ /** commander value-parser for `--log-format`. */
40
+ function parseLogFormat(value) {
41
+ const problem = logFormatProblem(value);
42
+ if (problem !== undefined)
43
+ throw new InvalidArgumentError(problem);
44
+ return value;
45
+ }
46
+ /**
47
+ * commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
48
+ * or "" when nothing is close: the same rule as commander's own (an optimal-string-
49
+ * alignment distance below 3 that keeps more than 40 % of the word). The root action
50
+ * reports an unknown command itself, so commander's suggestion is rebuilt here.
51
+ */
52
+ export function suggestSimilar(word, candidates) {
53
+ let similar = [];
54
+ let best = 3;
55
+ for (const candidate of new Set(candidates)) {
56
+ if (candidate.length <= 1)
57
+ continue;
58
+ const distance = editDistance(word, candidate, best);
59
+ const length = Math.max(word.length, candidate.length);
60
+ if ((length - distance) / length <= 0.4)
61
+ continue;
62
+ if (distance < best) {
63
+ best = distance;
64
+ similar = [candidate];
65
+ }
66
+ else if (distance === best) {
67
+ similar.push(candidate);
68
+ }
69
+ }
70
+ similar.sort((a, b) => a.localeCompare(b));
71
+ if (similar.length > 1)
72
+ return `\n(Did you mean one of ${similar.join(", ")}?)`;
73
+ return similar.length === 1 ? `\n(Did you mean ${similar[0]}?)` : "";
74
+ }
75
+ /** Optimal-string-alignment distance between `a` and `b`; at least `cap` when it is that far. */
76
+ function editDistance(a, b, cap) {
77
+ if (Math.abs(a.length - b.length) >= cap)
78
+ return Math.max(a.length, b.length);
79
+ const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
80
+ for (let j = 1; j <= b.length; j++)
81
+ d[0][j] = j;
82
+ for (let i = 1; i <= a.length; i++) {
83
+ for (let j = 1; j <= b.length; j++) {
84
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
85
+ let v = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
86
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1])
87
+ v = Math.min(v, d[i - 2][j - 2] + 1);
88
+ d[i][j] = v;
89
+ }
90
+ }
91
+ return d[a.length][b.length];
92
+ }
38
93
  export function buildProgram(deps = defaultDeps) {
39
94
  const program = new Command();
40
95
  const baseUrlDefault = deps.env["CKAN_BASE_URL"] || DEFAULT_BASE_URL;
@@ -43,17 +98,18 @@ export function buildProgram(deps = defaultDeps) {
43
98
  .description("CLI for any CKAN open-data portal (Action API v3)")
44
99
  .version(VERSION)
45
100
  .addOption(new Option("--base-url <url>", "CKAN site URL (env CKAN_BASE_URL)")
46
- .argParser(parseBaseUrl)
101
+ .argParser(once(parseBaseUrl))
47
102
  // The help shows the default without userinfo: a password in
48
103
  // CKAN_BASE_URL must not end up in `--help` output or CI logs.
49
104
  .default(baseUrlDefault, JSON.stringify(redactUrl(baseUrlDefault))))
50
105
  .addOption(new Option("--portal <id>", "a known portal by id (see `ckan portals`)")
51
- .argParser(parsePortal)
106
+ .argParser(once(parsePortal))
52
107
  .conflicts("baseUrl"))
53
- .option("--timeout <ms>", "per-request timeout in milliseconds", parseBoundedInt(0, MAX_TIMEOUT_MS))
54
- .option("--user-agent <ua>", "User-Agent header value", parseHeaderValue)
55
- .option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each waits the server's Retry-After, up to ${MAX_RETRY_AFTER_MS / 1000} s)`, parseBoundedInt(0, MAX_RETRIES))
56
- .option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", parseIntArg)
108
+ .option("--timeout <ms>", "per-request timeout in milliseconds", once(parseBoundedInt(0, MAX_TIMEOUT_MS)))
109
+ .option("--user-agent <ua>", "User-Agent header value", once(parseHeaderValue))
110
+ .option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each backs off linearly, or waits a longer Retry-After up to ${MAX_RETRY_AFTER_MS / 1000} s)`, once(parseBoundedInt(0, MAX_RETRIES)))
111
+ .option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", once(parseIntArg))
112
+ .option("--log-format <format>", `how errors, warnings and notes are written to stderr: text (log4j style: time, level, [topic], message) or jsonl (one JSON object per line: ts, level, topic, msg); default ${DEFAULT_LOG_FORMAT}`, once(parseLogFormat))
57
113
  .option("--compact", "print JSON on a single line instead of pretty-printed")
58
114
  .showHelpAfterError();
59
115
  // A bare invocation (no subcommand) should print help to stdout and exit 0,
@@ -68,13 +124,34 @@ export function buildProgram(deps = defaultDeps) {
68
124
  program.action(() => {
69
125
  const [unknown] = program.args;
70
126
  if (unknown !== undefined) {
71
- program.error(`error: unknown command '${unknown}'`, { code: "commander.unknownCommand" });
127
+ // The name is the user's: cut, after its credentials are redacted (a cut could
128
+ // otherwise leave part of a password without the "@" the redaction keys on).
129
+ const names = program.commands.map((c) => c.name()).filter((name) => name !== "help");
130
+ program.error(`error: unknown command '${cutForMessage(redactUrl(unknown))}'${suggestSimilar(unknown, names)}`, {
131
+ code: "commander.unknownCommand",
132
+ });
72
133
  }
73
134
  program.help();
74
135
  });
75
136
  // commander runs value parsers on flags but not on defaults, so a base URL
76
- // taken from CKAN_BASE_URL is checked here, before any command runs.
77
- program.hook("preAction", () => {
137
+ // taken from CKAN_BASE_URL is checked here, before a command that uses it runs.
138
+ // The commands that never use it skip the check: help (the root action prints
139
+ // help or names an unknown command, `help` prints help) and `portals`, which
140
+ // prints the built-in list or checks each portal at its own URL — a user whose
141
+ // variable is broken is exactly the one looking for a working portal id.
142
+ const offline = new Set(["help", "portals"]);
143
+ // One source for the log format once commander has parsed argv: its value, not the
144
+ // scan of argv (an option's value can look like --log-format; `--` ends the scan, not
145
+ // commander's parse of a value). The first hook, so the CKAN_BASE_URL check below
146
+ // already logs in that format.
147
+ program.hook("preAction", (_program, actionCommand) => {
148
+ const format = actionCommand.optsWithGlobals().logFormat;
149
+ if (deps.log !== undefined)
150
+ deps.log.format = format ?? DEFAULT_LOG_FORMAT;
151
+ });
152
+ program.hook("preAction", (_program, actionCommand) => {
153
+ if (actionCommand === program || offline.has(actionCommand.name()))
154
+ return;
78
155
  if (program.getOptionValueSource("baseUrl") !== "default" || program.opts()["portal"] !== undefined)
79
156
  return;
80
157
  try {
@@ -1,2 +1,54 @@
1
- import type { CliDeps } from "./io.js";
1
+ import { type Command } from "commander";
2
+ import { type CliDeps } from "./io.js";
3
+ import { type Logger } from "./log.js";
4
+ /**
5
+ * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
6
+ * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
7
+ * covers a URL that does not parse; a fallback behind the exact strings of
8
+ * `withRedactedOutput`.
9
+ */
10
+ export declare function redactUserinfo(text: string): string;
11
+ /** The secrets of a run, and the two ways they are replaced. */
12
+ export interface Redaction {
13
+ /**
14
+ * stdout text: the userinfo of every URL-like argument and of CKAN_BASE_URL replaced
15
+ * (`***@`), and the forms a server echoes it back in (the Basic value, the decoded
16
+ * `user:password`).
17
+ */
18
+ out(text: string): string;
19
+ /** stderr text, a record's message: that, and the password alone (`***`). */
20
+ err(text: string): string;
21
+ }
22
+ /**
23
+ * The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
24
+ * its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
25
+ * own messages name the values they refuse, and help shows the base URL's default:
26
+ * whatever path a credential takes to stdout or stderr, the exact userinfo (as
27
+ * `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
28
+ * by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
29
+ * `/`; the exact strings can. Without credentials the text passes through unchanged.
30
+ */
31
+ export declare function redactionFor(argv: readonly string[], env: Record<string, string | undefined>): Redaction;
32
+ /**
33
+ * `deps` that keep the secrets of this run (`redactionFor`) out of everything they
34
+ * print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
35
+ * message before formatting it, then writes to the raw `io.err`, so the frame is never
36
+ * touched and a password with DEL, C1 or bidi characters is matched in its raw form.
37
+ * `io.err` itself is redacted too, for anything that writes to stderr without the log.
38
+ */
39
+ export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[]): CliDeps;
40
+ /**
41
+ * The index of a `help` command whose topic names no command (`ckan help nonexistent`),
42
+ * or undefined. Commander answers that with the root help on stderr and exit 1, but
43
+ * no line saying what was wrong. Global options before `help` are skipped with their
44
+ * values; scanning stops at `--`.
45
+ */
46
+ export declare function unknownHelpTopic(program: Command, argv: readonly string[]): number | undefined;
47
+ /**
48
+ * The log for what happens outside `run()`, in the bin shim: a stdout write error
49
+ * (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
50
+ * for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
51
+ * run's own log; it writes to the raw stderr.
52
+ */
53
+ export declare function processLogger(argv: readonly string[], env?: Record<string, string | undefined>): Logger;
2
54
  export declare function run(argv: string[], deps?: CliDeps): Promise<number>;
@@ -3,7 +3,9 @@
3
3
  // captured output and exit code without spawning a subprocess.
4
4
  import { CommanderError } from "commander";
5
5
  import { buildProgram, defaultDeps } from "./program.js";
6
- import { CkanApiError, CkanError, CkanValidationError } from "../client/errors.js";
6
+ import { logOf } from "./io.js";
7
+ import { createLogger, logFormatFromArgv } from "./log.js";
8
+ import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "../client/errors.js";
7
9
  /** The exit code of a usage error: commander's own for a rejected option value. */
8
10
  const USAGE_EXIT = 1;
9
11
  /**
@@ -11,18 +13,241 @@ const USAGE_EXIT = 1;
11
13
  * commander does not propagate these to subcommands, so a parse error on a
12
14
  * subcommand would otherwise call process.exit() and bypass our error handling.
13
15
  */
14
- function configureTree(command, deps) {
16
+ function configureTree(command, deps, state = { errorLogged: false }) {
15
17
  command.exitOverride();
16
18
  command.configureOutput({
17
19
  writeOut: (str) => deps.io.out(str.replace(/\n$/, "")),
18
- writeErr: (str) => deps.io.err(str.replace(/\n$/, "")),
20
+ writeErr: (str) => writeCommanderErr(command, deps, state, str),
19
21
  });
20
22
  for (const child of command.commands)
21
- configureTree(child, deps);
23
+ configureTree(child, deps, state);
24
+ }
25
+ /**
26
+ * The names (long and short) of the program's own options that require a value
27
+ * (`--user-agent`). Only the program's: commander takes them out of argv wherever they
28
+ * stand, before a subcommand sees the rest, so a subcommand's `--rows` never swallows
29
+ * a `--log-format` after it.
30
+ */
31
+ function valueOptionsOf(program) {
32
+ const names = new Set();
33
+ for (const option of program.options) {
34
+ if (!option.required)
35
+ continue;
36
+ if (option.long !== undefined)
37
+ names.add(option.long);
38
+ if (option.short !== undefined)
39
+ names.add(option.short);
40
+ }
41
+ return names;
42
+ }
43
+ /** `ckan search`: the command's name with its parents'. */
44
+ function commandPath(command) {
45
+ const names = [];
46
+ for (let c = command; c !== null; c = c.parent)
47
+ names.unshift(c.name());
48
+ return names.join(" ");
49
+ }
50
+ /**
51
+ * commander's stderr output as log records, one per line. Its `error: …` is an ERROR of
52
+ * `cli`, with a following `(Did you mean …?)` line appended to that same record; the
53
+ * help it shows after an error is one INFO record per non-blank line. A command group
54
+ * run without its subcommand makes commander show the help as an error with no
55
+ * `error:` line: an ERROR record "missing command: `ckan <group> <subcommand>`" comes
56
+ * first, so every failed run has one (ckan has no groups today; a bare `ckan` prints
57
+ * its help to stdout and exits 0).
58
+ */
59
+ function writeCommanderErr(command, deps, state, str) {
60
+ const log = logOf(deps);
61
+ const text = str.replace(/\n$/, "");
62
+ // The blank line commander writes between an error and the help it shows after.
63
+ if (text.trim() === "")
64
+ return;
65
+ if (text.startsWith("error: ")) {
66
+ state.errorLogged = true;
67
+ log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
68
+ return;
69
+ }
70
+ if (!state.errorLogged) {
71
+ state.errorLogged = true;
72
+ log.error("cli", `missing command: \`${commandPath(command)} <subcommand>\``);
73
+ }
74
+ for (const line of text.split("\n"))
75
+ if (line.trim() !== "")
76
+ log.info("cli", line.trimEnd());
77
+ }
78
+ /**
79
+ * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
80
+ * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
81
+ * covers a URL that does not parse; a fallback behind the exact strings of
82
+ * `withRedactedOutput`.
83
+ */
84
+ export function redactUserinfo(text) {
85
+ return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
86
+ }
87
+ /**
88
+ * The options whose value is a base URL (`--portal` takes a portal's URL as well as its
89
+ * id): a `user:password@host` given there without its scheme is still a credential, like
90
+ * one in CKAN_BASE_URL (anywhere else a bare `a:b@c` is not).
91
+ */
92
+ const BASE_URL_FLAGS = ["--base-url", "--portal"];
93
+ /** The values of the `flags` in `argv`, in both forms (`--flag value`, `--flag=value`). */
94
+ function flagValues(argv, flags) {
95
+ const found = [];
96
+ argv.forEach((token, i) => {
97
+ const next = argv[i + 1];
98
+ if (flags.includes(token) && next !== undefined)
99
+ found.push(next);
100
+ const eq = token.indexOf("=");
101
+ if (eq > 0 && flags.includes(token.slice(0, eq)))
102
+ found.push(token.slice(eq + 1));
103
+ });
104
+ return found;
105
+ }
106
+ /**
107
+ * The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
108
+ * its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
109
+ * own messages name the values they refuse, and help shows the base URL's default:
110
+ * whatever path a credential takes to stdout or stderr, the exact userinfo (as
111
+ * `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
112
+ * by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
113
+ * `/`; the exact strings can. Without credentials the text passes through unchanged.
114
+ */
115
+ export function redactionFor(argv, env) {
116
+ // An `--option=value` token is echoed as its value alone.
117
+ const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
118
+ // A base URL typed without its scheme is read as if it had one (anywhere else a bare
119
+ // `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
120
+ const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
121
+ const sources = [...values, ...baseUrls];
122
+ const secrets = new Set();
123
+ const encoded = new Set();
124
+ const echoed = new Set();
125
+ const passwords = new Set();
126
+ for (const source of sources) {
127
+ for (const secret of credentialsIn(source)) {
128
+ secrets.add(secret);
129
+ secrets.add(JSON.stringify(secret).slice(1, -1));
130
+ // A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
131
+ // 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
132
+ encoded.add(encodeURIComponent(`${secret}@`));
133
+ // What a server echoes back: the Basic value and the decoded user:password on
134
+ // stdout and stderr, the password alone (it may well occur in the data) on stderr.
135
+ const [basic, pair, password] = echoedCredentialForms(secret);
136
+ if (basic !== undefined)
137
+ echoed.add(basic);
138
+ if (pair !== undefined)
139
+ echoed.add(pair);
140
+ if (password !== undefined)
141
+ passwords.add(password);
142
+ }
143
+ }
144
+ if (secrets.size === 0)
145
+ return { out: (text) => text, err: (text) => text };
146
+ const list = [...secrets];
147
+ // Longest first, so a secret is never left half-replaced by one of its own substrings.
148
+ const echoedList = [...echoed].sort((a, b) => b.length - a.length);
149
+ const passwordList = [...passwords].sort((a, b) => b.length - a.length);
150
+ const out = (text) => {
151
+ let result = redactUserinfo(redactCredentials(text, list));
152
+ for (const form of encoded)
153
+ result = result.split(form).join("***%40");
154
+ return redactSecrets(result, echoedList);
155
+ };
156
+ return { out, err: (text) => redactSecrets(out(text), passwordList) };
157
+ }
158
+ /**
159
+ * `deps` that keep the secrets of this run (`redactionFor`) out of everything they
160
+ * print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
161
+ * message before formatting it, then writes to the raw `io.err`, so the frame is never
162
+ * touched and a password with DEL, C1 or bidi characters is matched in its raw form.
163
+ * `io.err` itself is redacted too, for anything that writes to stderr without the log.
164
+ */
165
+ export function withRedactedOutput(deps, argv) {
166
+ const redaction = redactionFor(argv, deps.env);
167
+ const { out, err } = deps.io;
168
+ return {
169
+ ...deps,
170
+ io: { ...deps.io, out: (text) => out(redaction.out(text)), err: (text) => err(redaction.err(text)) },
171
+ log: createLogger({
172
+ format: logFormatFromArgv(argv),
173
+ write: err,
174
+ redact: redaction.err,
175
+ ...(deps.now === undefined ? {} : { now: deps.now }),
176
+ }),
177
+ };
178
+ }
179
+ /**
180
+ * The index of a `help` command whose topic names no command (`ckan help nonexistent`),
181
+ * or undefined. Commander answers that with the root help on stderr and exit 1, but
182
+ * no line saying what was wrong. Global options before `help` are skipped with their
183
+ * values; scanning stops at `--`.
184
+ */
185
+ export function unknownHelpTopic(program, argv) {
186
+ for (let i = 0; i < argv.length; i++) {
187
+ const token = argv[i];
188
+ if (token === "--")
189
+ return undefined;
190
+ if (token.startsWith("-")) {
191
+ if (token.includes("="))
192
+ continue;
193
+ const option = program.options.find((o) => o.long === token || o.short === token);
194
+ if (option?.required)
195
+ i++;
196
+ continue;
197
+ }
198
+ if (token !== "help")
199
+ return undefined;
200
+ const topic = argv[i + 1];
201
+ if (topic === undefined || topic.startsWith("-"))
202
+ return undefined;
203
+ const known = program.commands.some((c) => c.name() === topic || c.aliases().includes(topic));
204
+ return known ? undefined : i;
205
+ }
206
+ return undefined;
207
+ }
208
+ /**
209
+ * The log for what happens outside `run()`, in the bin shim: a stdout write error
210
+ * (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
211
+ * for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
212
+ * run's own log; it writes to the raw stderr.
213
+ */
214
+ export function processLogger(argv, env = process.env) {
215
+ return createLogger({
216
+ format: logFormatFromArgv(argv),
217
+ write: (line) => process.stderr.write(line + "\n"),
218
+ redact: redactionFor(argv, env).err,
219
+ });
220
+ }
221
+ /**
222
+ * The log area of a `CkanError` that is neither an API error nor a usage error: the
223
+ * connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
224
+ * envelope, the wrong result shape, an unknown charset — the portal's answer as much as
225
+ * an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
226
+ * envelope, `CkanActionError`), else `cli` (a response nested too deeply to print is
227
+ * about printing the answer, not its shape).
228
+ */
229
+ function areaOf(err) {
230
+ if (err instanceof CkanNetworkError)
231
+ return "http";
232
+ if (err instanceof CkanParseError || err instanceof CkanActionError)
233
+ return "api";
234
+ return "cli";
22
235
  }
23
236
  export async function run(argv, deps = defaultDeps) {
237
+ // The log replaces the secrets of the run in every message, in either format.
238
+ deps = withRedactedOutput(deps, argv);
24
239
  const program = buildProgram(deps);
25
240
  configureTree(program, deps);
241
+ // For the records of a parse error: the scan of argv, now knowing which of the program's
242
+ // options take a value, as commander reads them. Once commander has parsed argv, the program's
243
+ // first preAction hook sets the format it parsed (`buildProgram`).
244
+ if (deps.log !== undefined)
245
+ deps.log.format = logFormatFromArgv(argv, valueOptionsOf(program));
246
+ // `help <unknown>` is answered like `<unknown>`: "error: unknown command '<unknown>'",
247
+ // the help after it, exit 1 — not the help alone with a failure code.
248
+ const helpAt = unknownHelpTopic(program, argv);
249
+ if (helpAt !== undefined)
250
+ argv = argv.filter((_, i) => i !== helpAt);
26
251
  try {
27
252
  await program.parseAsync(argv, { from: "user" });
28
253
  return 0;
@@ -32,24 +257,30 @@ export async function run(argv, deps = defaultDeps) {
32
257
  // Help/version requests exit 0; genuine parse errors carry their own code.
33
258
  return err.exitCode;
34
259
  }
260
+ const log = logOf(deps);
35
261
  if (err instanceof CkanValidationError) {
36
262
  // An input the library refused before any request (a rule the commander
37
263
  // parsers do not see, e.g. one across parameters): a usage error.
38
- deps.io.err(`Error: ${err.message}`);
264
+ log.error("cli", err.message);
39
265
  return USAGE_EXIT;
40
266
  }
41
267
  if (err instanceof CkanApiError) {
42
- deps.io.err(`Error: ${err.message}`);
268
+ log.error("api", err.message);
43
269
  // Map a few notable statuses to distinct exit codes for scripting.
44
270
  if (err.status === 404)
45
271
  return 4;
46
272
  return 1;
47
273
  }
274
+ if (err instanceof CkanActionError) {
275
+ log.error(areaOf(err), err.message);
276
+ // CKAN's Not Found Error is a not-found whatever the HTTP status it came with.
277
+ return err.isNotFound ? 4 : 1;
278
+ }
48
279
  if (err instanceof CkanError) {
49
- deps.io.err(`Error: ${err.message}`);
280
+ log.error(areaOf(err), err.message);
50
281
  return 1;
51
282
  }
52
- deps.io.err(`Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
283
+ log.error("cli", `Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
53
284
  return 1;
54
285
  }
55
286
  }