@maschinenlesbar.org/ckan-cli 0.2.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,159 @@
1
+ // The CLI's log: every diagnostic line on stderr — errors, warnings, notes — is one
2
+ // record with a timestamp, a level and a topic. Two formats, chosen with the global
3
+ // `--log-format`:
4
+ //
5
+ // text 2026-10-09T14:03:12.481Z WARN [ckan.http] requests to … are sent unencrypted …
6
+ // jsonl {"ts":"2026-10-09T14:03:12.481Z","level":"WARN","topic":"ckan.http","msg":"…"}
7
+ //
8
+ // The text form follows log4j's pattern (`%d %-5p [%c] %m`), with the time in UTC
9
+ // ISO 8601. The topic is a dotted logger name: the program, then the area the record
10
+ // comes from (`ckan.cli`, `ckan.api`, `ckan.http`). stdout carries data
11
+ // only and is not touched; neither is `--help`/`--version`.
12
+ import { cutText, toWellFormed } from "../client/errors.js";
13
+ /** The log formats `--log-format` takes. */
14
+ export const LOG_FORMATS = ["text", "jsonl"];
15
+ /** The format without `--log-format`. */
16
+ export const DEFAULT_LOG_FORMAT = "text";
17
+ /** The program's name in every topic: `ckan.<area>`. */
18
+ export const LOG_PROGRAM = "ckan";
19
+ /** True for a character a record never carries raw (see `escapeForRecord`). */
20
+ function escapedInRecords(c) {
21
+ return ((c < 0x20 && c !== 0x09) || // C0 but TAB
22
+ (c >= 0x7f && c <= 0x9f) || // DEL and C1 (NEL, the 8-bit CSI)
23
+ c === 0x2028 || c === 0x2029 || // line and paragraph separator
24
+ c === 0x061c || c === 0x200e || c === 0x200f || // bidi marks
25
+ (c >= 0x202a && c <= 0x202e) || // bidi embeddings and overrides
26
+ (c >= 0x2066 && c <= 0x2069) // bidi isolates
27
+ );
28
+ }
29
+ /**
30
+ * `text` with every character that could split a record, forge a second one or steer
31
+ * the terminal written as an escape: CR as `\r`, LF as `\n`, any other C0 control but
32
+ * TAB, DEL and C1 as `\u00XX`, U+2028, U+2029 and the bidi controls (U+061C, U+200E,
33
+ * U+200F, U+202A–U+202E, U+2066–U+2069) as `\uXXXX`. Backslashes stay as they are. On
34
+ * the output of `JSON.stringify` (no raw C0 left) every escape it adds is valid JSON,
35
+ * so the same helper serves both formats. Checked by char code, so the source stays
36
+ * free of those characters.
37
+ */
38
+ export function escapeForRecord(text) {
39
+ let out = "";
40
+ let from = 0;
41
+ for (let i = 0; i < text.length; i++) {
42
+ const c = text.charCodeAt(i);
43
+ if (!escapedInRecords(c))
44
+ continue;
45
+ const escaped = c === 0x0d ? "\\r" : c === 0x0a ? "\\n" : "\\u" + c.toString(16).padStart(4, "0");
46
+ out += text.slice(from, i) + escaped;
47
+ from = i + 1;
48
+ }
49
+ return from === 0 ? text : out + text.slice(from);
50
+ }
51
+ /**
52
+ * The longest message (in characters) a record carries. A longer one is cut at a
53
+ * code-point boundary and ends in `… (N more characters)`: one huge server text or typed
54
+ * value cannot flood stderr or a log store with a line of megabytes. Own messages that
55
+ * quote server or user data are bounded at their source too (`cutForMessage`); this
56
+ * is the backstop for every path.
57
+ */
58
+ export const MAX_RECORD_MESSAGE = 4000;
59
+ /** `msg` within `MAX_RECORD_MESSAGE`, the cut marked with the number of characters left out. */
60
+ function boundRecordMessage(msg) {
61
+ if (msg.length <= MAX_RECORD_MESSAGE)
62
+ return msg;
63
+ const kept = cutText(msg, MAX_RECORD_MESSAGE);
64
+ return `${kept}… (${codePoints(msg, kept.length)} more characters)`;
65
+ }
66
+ /** The number of characters (code points) in `text` from `from` on. */
67
+ function codePoints(text, from) {
68
+ let n = 0;
69
+ for (let i = from; i < text.length; i++) {
70
+ const c = text.charCodeAt(i);
71
+ // The low half of a pair is not a character of its own.
72
+ if (c >= 0xdc00 && c <= 0xdfff && i > from && text.charCodeAt(i - 1) >= 0xd800 && text.charCodeAt(i - 1) <= 0xdbff)
73
+ continue;
74
+ n++;
75
+ }
76
+ return n;
77
+ }
78
+ /**
79
+ * One record as one line, whatever the message holds: `escapeForRecord` runs over the
80
+ * message (text) or over the whole JSON object (jsonl), so no text that reaches a
81
+ * record can split it, forge another one, or reach the terminal as a control sequence.
82
+ */
83
+ export function formatLogRecord(record, format) {
84
+ // Well-formed first: half a character would be `\ud83d` in jsonl, which jq rejects,
85
+ // stopping the whole stream.
86
+ const msg = toWellFormed(boundRecordMessage(record.msg));
87
+ if (format === "jsonl")
88
+ return escapeForRecord(JSON.stringify({ ts: record.ts, level: record.level, topic: record.topic, msg }));
89
+ return `${record.ts} ${record.level.padEnd(5)} [${record.topic}] ${escapeForRecord(msg)}`;
90
+ }
91
+ /**
92
+ * A logger that writes each record, formatted, to `write` (stderr). `redact` replaces the
93
+ * secrets of the run in the message, before it is cut and escaped: the frame (time,
94
+ * level, topic) is never touched, so a secret equal to a year or a topic cannot corrupt
95
+ * it, and a secret holding DEL, C1 or bidi characters is found in its raw form.
96
+ */
97
+ export function createLogger(options) {
98
+ const now = options.now ?? (() => new Date());
99
+ const redact = options.redact ?? ((text) => text);
100
+ const logger = {
101
+ format: options.format,
102
+ error: (area, msg) => log("ERROR", area, msg),
103
+ warn: (area, msg) => log("WARN", area, msg),
104
+ info: (area, msg) => log("INFO", area, msg),
105
+ };
106
+ const log = (level, area, msg) => options.write(formatLogRecord({ ts: now().toISOString(), level, topic: `${LOG_PROGRAM}.${area}`, msg: redact(msg) }, logger.format));
107
+ return logger;
108
+ }
109
+ /** Why `value` is not a log format, or undefined. */
110
+ export function logFormatProblem(value) {
111
+ return LOG_FORMATS.includes(value) ? undefined : `Expected one of ${LOG_FORMATS.join(", ")}.`;
112
+ }
113
+ /**
114
+ * The `--log-format` in `argv`, read before commander parses it: commander's own
115
+ * usage errors are logged too, and they happen while parsing. A missing or unknown
116
+ * value gives the default here; commander then reports an unknown one. The first
117
+ * `--log-format` counts: commander keeps it and rejects a second one (`once`), so that
118
+ * error is logged in the first one's format. This scan is only for the records of a
119
+ * parse error: once commander has parsed argv, its value is the format (the program's
120
+ * first preAction hook), so `--user-agent --log-format=jsonl` (a User-Agent) logs text.
121
+ * `valueOptions` names the program's options that take a value (`--user-agent`): the
122
+ * token after one is its value, never an option, as commander reads it, so `--user-agent
123
+ * --log-format jsonl` agrees with commander in a parse error too. A subcommand's value
124
+ * option (`--rows`) does not count: commander takes the program's own options out of
125
+ * argv first, so `search --rows --log-format jsonl` is jsonl and a `--rows` without its
126
+ * value.
127
+ */
128
+ export function logFormatFromArgv(argv, valueOptions = new Set()) {
129
+ let format;
130
+ for (let i = 0; i < argv.length; i++) {
131
+ const token = argv[i];
132
+ if (token === "--")
133
+ break;
134
+ if (token !== "--log-format" && valueOptions.has(token)) {
135
+ i++;
136
+ continue;
137
+ }
138
+ if (token === "--log-format")
139
+ format = argv[i + 1];
140
+ else if (token.startsWith("--log-format="))
141
+ format = token.slice("--log-format=".length);
142
+ else
143
+ continue;
144
+ break;
145
+ }
146
+ return format !== undefined && logFormatProblem(format) === undefined ? format : DEFAULT_LOG_FORMAT;
147
+ }
148
+ /**
149
+ * Node's own process warnings (`(node:PID) Warning: …`, e.g. for
150
+ * `NODE_TLS_REJECT_UNAUTHORIZED=0`) as WARN records of `ckan.cli`:
151
+ * `(node) <name>: <message>`, in the log's format and with its redaction. Node's default
152
+ * listener, which prints the plain line, is removed (checked on Node 22, 24 and 26: that
153
+ * silences it; `--no-warnings` is not needed). The bin shim installs it once, before
154
+ * `run()`; a warning Node emits while it starts up, before any code runs, stays Node's.
155
+ */
156
+ export function installWarningLog(source, log) {
157
+ source.removeAllListeners("warning");
158
+ source.on("warning", (warning) => log.warn("cli", `(node) ${warning.name}: ${warning.message}`));
159
+ }
@@ -3,4 +3,11 @@ import type { CliDeps } from "./io.js";
3
3
  export declare const VERSION: string;
4
4
  /** Default dependencies: real client + real stdout/stderr + real environment. */
5
5
  export declare const defaultDeps: CliDeps;
6
+ /**
7
+ * commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
8
+ * or "" when nothing is close: the same rule as commander's own (an optimal-string-
9
+ * alignment distance below 3 that keeps more than 40 % of the word). The root action
10
+ * reports an unknown command itself, so commander's suggestion is rebuilt here.
11
+ */
12
+ export declare function suggestSimilar(word: string, candidates: readonly string[]): string;
6
13
  export declare function buildProgram(deps?: CliDeps): Command;
@@ -8,10 +8,11 @@ import { defaultIO } from "./io.js";
8
8
  import { CkanClient } from "../client/client.js";
9
9
  import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
10
10
  import { MAX_TIMEOUT_MS } from "../client/http.js";
11
- import { redactUrl } from "../client/errors.js";
11
+ import { cutForMessage, redactUrl } from "../client/errors.js";
12
12
  import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
13
13
  import { registerCatalogueCommands } from "./commands/catalogue.js";
14
14
  import { registerPortalCommands } from "./commands/portal.js";
15
+ import { DEFAULT_LOG_FORMAT, logFormatProblem } from "./log.js";
15
16
  /**
16
17
  * Single source of truth for the version: read from package.json at runtime
17
18
  * rather than duplicating a literal that can silently drift after a release bump.
@@ -35,6 +36,60 @@ export const defaultDeps = {
35
36
  env: process.env,
36
37
  createClient: (options) => new CkanClient(options),
37
38
  };
39
+ /** commander value-parser for `--log-format`. */
40
+ function parseLogFormat(value) {
41
+ const problem = logFormatProblem(value);
42
+ if (problem !== undefined)
43
+ throw new InvalidArgumentError(problem);
44
+ return value;
45
+ }
46
+ /**
47
+ * commander's hint for a mistyped command, `\n(Did you mean status?)` (or `one of a, b`),
48
+ * or "" when nothing is close: the same rule as commander's own (an optimal-string-
49
+ * alignment distance below 3 that keeps more than 40 % of the word). The root action
50
+ * reports an unknown command itself, so commander's suggestion is rebuilt here.
51
+ */
52
+ export function suggestSimilar(word, candidates) {
53
+ let similar = [];
54
+ let best = 3;
55
+ for (const candidate of new Set(candidates)) {
56
+ if (candidate.length <= 1)
57
+ continue;
58
+ const distance = editDistance(word, candidate, best);
59
+ const length = Math.max(word.length, candidate.length);
60
+ if ((length - distance) / length <= 0.4)
61
+ continue;
62
+ if (distance < best) {
63
+ best = distance;
64
+ similar = [candidate];
65
+ }
66
+ else if (distance === best) {
67
+ similar.push(candidate);
68
+ }
69
+ }
70
+ similar.sort((a, b) => a.localeCompare(b));
71
+ if (similar.length > 1)
72
+ return `\n(Did you mean one of ${similar.join(", ")}?)`;
73
+ return similar.length === 1 ? `\n(Did you mean ${similar[0]}?)` : "";
74
+ }
75
+ /** Optimal-string-alignment distance between `a` and `b`; at least `cap` when it is that far. */
76
+ function editDistance(a, b, cap) {
77
+ if (Math.abs(a.length - b.length) >= cap)
78
+ return Math.max(a.length, b.length);
79
+ const d = Array.from({ length: a.length + 1 }, (_, i) => [i, ...Array(b.length).fill(0)]);
80
+ for (let j = 1; j <= b.length; j++)
81
+ d[0][j] = j;
82
+ for (let i = 1; i <= a.length; i++) {
83
+ for (let j = 1; j <= b.length; j++) {
84
+ const cost = a[i - 1] === b[j - 1] ? 0 : 1;
85
+ let v = Math.min(d[i - 1][j] + 1, d[i][j - 1] + 1, d[i - 1][j - 1] + cost);
86
+ if (i > 1 && j > 1 && a[i - 1] === b[j - 2] && a[i - 2] === b[j - 1])
87
+ v = Math.min(v, d[i - 2][j - 2] + 1);
88
+ d[i][j] = v;
89
+ }
90
+ }
91
+ return d[a.length][b.length];
92
+ }
38
93
  export function buildProgram(deps = defaultDeps) {
39
94
  const program = new Command();
40
95
  const baseUrlDefault = deps.env["CKAN_BASE_URL"] || DEFAULT_BASE_URL;
@@ -54,6 +109,7 @@ export function buildProgram(deps = defaultDeps) {
54
109
  .option("--user-agent <ua>", "User-Agent header value", once(parseHeaderValue))
55
110
  .option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each backs off linearly, or waits a longer Retry-After up to ${MAX_RETRY_AFTER_MS / 1000} s)`, once(parseBoundedInt(0, MAX_RETRIES)))
56
111
  .option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", once(parseIntArg))
112
+ .option("--log-format <format>", `how errors, warnings and notes are written to stderr: text (log4j style: time, level, [topic], message) or jsonl (one JSON object per line: ts, level, topic, msg); default ${DEFAULT_LOG_FORMAT}`, once(parseLogFormat))
57
113
  .option("--compact", "print JSON on a single line instead of pretty-printed")
58
114
  .showHelpAfterError();
59
115
  // A bare invocation (no subcommand) should print help to stdout and exit 0,
@@ -68,7 +124,12 @@ export function buildProgram(deps = defaultDeps) {
68
124
  program.action(() => {
69
125
  const [unknown] = program.args;
70
126
  if (unknown !== undefined) {
71
- program.error(`error: unknown command '${unknown}'`, { code: "commander.unknownCommand" });
127
+ // The name is the user's: cut, after its credentials are redacted (a cut could
128
+ // otherwise leave part of a password without the "@" the redaction keys on).
129
+ const names = program.commands.map((c) => c.name()).filter((name) => name !== "help");
130
+ program.error(`error: unknown command '${cutForMessage(redactUrl(unknown))}'${suggestSimilar(unknown, names)}`, {
131
+ code: "commander.unknownCommand",
132
+ });
72
133
  }
73
134
  program.help();
74
135
  });
@@ -79,6 +140,15 @@ export function buildProgram(deps = defaultDeps) {
79
140
  // prints the built-in list or checks each portal at its own URL — a user whose
80
141
  // variable is broken is exactly the one looking for a working portal id.
81
142
  const offline = new Set(["help", "portals"]);
143
+ // One source for the log format once commander has parsed argv: its value, not the
144
+ // scan of argv (an option's value can look like --log-format; `--` ends the scan, not
145
+ // commander's parse of a value). The first hook, so the CKAN_BASE_URL check below
146
+ // already logs in that format.
147
+ program.hook("preAction", (_program, actionCommand) => {
148
+ const format = actionCommand.optsWithGlobals().logFormat;
149
+ if (deps.log !== undefined)
150
+ deps.log.format = format ?? DEFAULT_LOG_FORMAT;
151
+ });
82
152
  program.hook("preAction", (_program, actionCommand) => {
83
153
  if (actionCommand === program || offline.has(actionCommand.name()))
84
154
  return;
@@ -1,5 +1,6 @@
1
1
  import { type Command } from "commander";
2
- import type { CliDeps } from "./io.js";
2
+ import { type CliDeps } from "./io.js";
3
+ import { type Logger } from "./log.js";
3
4
  /**
4
5
  * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
5
6
  * (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
@@ -7,16 +8,33 @@ import type { CliDeps } from "./io.js";
7
8
  * `withRedactedOutput`.
8
9
  */
9
10
  export declare function redactUserinfo(text: string): string;
11
+ /** The secrets of a run, and the two ways they are replaced. */
12
+ export interface Redaction {
13
+ /**
14
+ * stdout text: the userinfo of every URL-like argument and of CKAN_BASE_URL replaced
15
+ * (`***@`), and the forms a server echoes it back in (the Basic value, the decoded
16
+ * `user:password`).
17
+ */
18
+ out(text: string): string;
19
+ /** stderr text, a record's message: that, and the password alone (`***`). */
20
+ err(text: string): string;
21
+ }
10
22
  /**
11
- * `deps` with an `io` that redacts the credentials of every argument and of
12
- * CKAN_BASE_URL from everything it prints. Commander echoes rejected values in its
13
- * errors (`--base-url`, `--portal`, a URL typed where the command goes), the
14
- * client's own messages name the values they refuse, and help shows the base URL's
15
- * default: whatever path a credential takes to stdout or stderr, the exact userinfo
16
- * (as `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is
17
- * replaced by `***`. A pattern alone can't delimit a password with spaces, quotes,
18
- * `#`, `?` or `/`; the exact strings can. Without credentials the output passes
19
- * through unchanged.
23
+ * The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
24
+ * its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
25
+ * own messages name the values they refuse, and help shows the base URL's default:
26
+ * whatever path a credential takes to stdout or stderr, the exact userinfo (as
27
+ * `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
28
+ * by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
29
+ * `/`; the exact strings can. Without credentials the text passes through unchanged.
30
+ */
31
+ export declare function redactionFor(argv: readonly string[], env: Record<string, string | undefined>): Redaction;
32
+ /**
33
+ * `deps` that keep the secrets of this run (`redactionFor`) out of everything they
34
+ * print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
35
+ * message before formatting it, then writes to the raw `io.err`, so the frame is never
36
+ * touched and a password with DEL, C1 or bidi characters is matched in its raw form.
37
+ * `io.err` itself is redacted too, for anything that writes to stderr without the log.
20
38
  */
21
39
  export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[]): CliDeps;
22
40
  /**
@@ -26,4 +44,11 @@ export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[
26
44
  * values; scanning stops at `--`.
27
45
  */
28
46
  export declare function unknownHelpTopic(program: Command, argv: readonly string[]): number | undefined;
47
+ /**
48
+ * The log for what happens outside `run()`, in the bin shim: a stdout write error
49
+ * (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
50
+ * for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
51
+ * run's own log; it writes to the raw stderr.
52
+ */
53
+ export declare function processLogger(argv: readonly string[], env?: Record<string, string | undefined>): Logger;
29
54
  export declare function run(argv: string[], deps?: CliDeps): Promise<number>;
@@ -3,7 +3,9 @@
3
3
  // captured output and exit code without spawning a subprocess.
4
4
  import { CommanderError } from "commander";
5
5
  import { buildProgram, defaultDeps } from "./program.js";
6
- import { CkanApiError, CkanError, CkanValidationError, credentialsIn, redactCredentials, } from "../client/errors.js";
6
+ import { logOf } from "./io.js";
7
+ import { createLogger, logFormatFromArgv } from "./log.js";
8
+ import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "../client/errors.js";
7
9
  /** The exit code of a usage error: commander's own for a rejected option value. */
8
10
  const USAGE_EXIT = 1;
9
11
  /**
@@ -11,14 +13,67 @@ const USAGE_EXIT = 1;
11
13
  * commander does not propagate these to subcommands, so a parse error on a
12
14
  * subcommand would otherwise call process.exit() and bypass our error handling.
13
15
  */
14
- function configureTree(command, deps) {
16
+ function configureTree(command, deps, state = { errorLogged: false }) {
15
17
  command.exitOverride();
16
18
  command.configureOutput({
17
19
  writeOut: (str) => deps.io.out(str.replace(/\n$/, "")),
18
- writeErr: (str) => deps.io.err(str.replace(/\n$/, "")),
20
+ writeErr: (str) => writeCommanderErr(command, deps, state, str),
19
21
  });
20
22
  for (const child of command.commands)
21
- configureTree(child, deps);
23
+ configureTree(child, deps, state);
24
+ }
25
+ /**
26
+ * The names (long and short) of the program's own options that require a value
27
+ * (`--user-agent`). Only the program's: commander takes them out of argv wherever they
28
+ * stand, before a subcommand sees the rest, so a subcommand's `--rows` never swallows
29
+ * a `--log-format` after it.
30
+ */
31
+ function valueOptionsOf(program) {
32
+ const names = new Set();
33
+ for (const option of program.options) {
34
+ if (!option.required)
35
+ continue;
36
+ if (option.long !== undefined)
37
+ names.add(option.long);
38
+ if (option.short !== undefined)
39
+ names.add(option.short);
40
+ }
41
+ return names;
42
+ }
43
+ /** `ckan search`: the command's name with its parents'. */
44
+ function commandPath(command) {
45
+ const names = [];
46
+ for (let c = command; c !== null; c = c.parent)
47
+ names.unshift(c.name());
48
+ return names.join(" ");
49
+ }
50
+ /**
51
+ * commander's stderr output as log records, one per line. Its `error: …` is an ERROR of
52
+ * `cli`, with a following `(Did you mean …?)` line appended to that same record; the
53
+ * help it shows after an error is one INFO record per non-blank line. A command group
54
+ * run without its subcommand makes commander show the help as an error with no
55
+ * `error:` line: an ERROR record "missing command: `ckan <group> <subcommand>`" comes
56
+ * first, so every failed run has one (ckan has no groups today; a bare `ckan` prints
57
+ * its help to stdout and exits 0).
58
+ */
59
+ function writeCommanderErr(command, deps, state, str) {
60
+ const log = logOf(deps);
61
+ const text = str.replace(/\n$/, "");
62
+ // The blank line commander writes between an error and the help it shows after.
63
+ if (text.trim() === "")
64
+ return;
65
+ if (text.startsWith("error: ")) {
66
+ state.errorLogged = true;
67
+ log.error("cli", text.slice("error: ".length).replace(/\n(\(Did you mean .*\?\))$/, " $1"));
68
+ return;
69
+ }
70
+ if (!state.errorLogged) {
71
+ state.errorLogged = true;
72
+ log.error("cli", `missing command: \`${commandPath(command)} <subcommand>\``);
73
+ }
74
+ for (const line of text.split("\n"))
75
+ if (line.trim() !== "")
76
+ log.info("cli", line.trimEnd());
22
77
  }
23
78
  /**
24
79
  * Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
@@ -30,22 +85,44 @@ export function redactUserinfo(text) {
30
85
  return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
31
86
  }
32
87
  /**
33
- * `deps` with an `io` that redacts the credentials of every argument and of
34
- * CKAN_BASE_URL from everything it prints. Commander echoes rejected values in its
35
- * errors (`--base-url`, `--portal`, a URL typed where the command goes), the
36
- * client's own messages name the values they refuse, and help shows the base URL's
37
- * default: whatever path a credential takes to stdout or stderr, the exact userinfo
38
- * (as `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is
39
- * replaced by `***`. A pattern alone can't delimit a password with spaces, quotes,
40
- * `#`, `?` or `/`; the exact strings can. Without credentials the output passes
41
- * through unchanged.
88
+ * The options whose value is a base URL (`--portal` takes a portal's URL as well as its
89
+ * id): a `user:password@host` given there without its scheme is still a credential, like
90
+ * one in CKAN_BASE_URL (anywhere else a bare `a:b@c` is not).
42
91
  */
43
- export function withRedactedOutput(deps, argv) {
92
+ const BASE_URL_FLAGS = ["--base-url", "--portal"];
93
+ /** The values of the `flags` in `argv`, in both forms (`--flag value`, `--flag=value`). */
94
+ function flagValues(argv, flags) {
95
+ const found = [];
96
+ argv.forEach((token, i) => {
97
+ const next = argv[i + 1];
98
+ if (flags.includes(token) && next !== undefined)
99
+ found.push(next);
100
+ const eq = token.indexOf("=");
101
+ if (eq > 0 && flags.includes(token.slice(0, eq)))
102
+ found.push(token.slice(eq + 1));
103
+ });
104
+ return found;
105
+ }
106
+ /**
107
+ * The secrets of the run in `argv` and CKAN_BASE_URL. Commander echoes rejected values in
108
+ * its errors (`--base-url`, `--portal`, a URL typed where the command goes), the client's
109
+ * own messages name the values they refuse, and help shows the base URL's default:
110
+ * whatever path a credential takes to stdout or stderr, the exact userinfo (as
111
+ * `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is replaced
112
+ * by `***`. A pattern alone can't delimit a password with spaces, quotes, `#`, `?` or
113
+ * `/`; the exact strings can. Without credentials the text passes through unchanged.
114
+ */
115
+ export function redactionFor(argv, env) {
44
116
  // An `--option=value` token is echoed as its value alone.
45
117
  const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
46
- const sources = [...argv, ...values, deps.env["CKAN_BASE_URL"] ?? ""];
118
+ // A base URL typed without its scheme is read as if it had one (anywhere else a bare
119
+ // `a:b@c` is no credential: a dataset id, a search text, a User-Agent).
120
+ const baseUrls = [...flagValues(argv, BASE_URL_FLAGS), env["CKAN_BASE_URL"] ?? ""].map((value) => value === "" || /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.test(value) ? value : `http://${value}`);
121
+ const sources = [...values, ...baseUrls];
47
122
  const secrets = new Set();
48
123
  const encoded = new Set();
124
+ const echoed = new Set();
125
+ const passwords = new Set();
49
126
  for (const source of sources) {
50
127
  for (const secret of credentialsIn(source)) {
51
128
  secrets.add(secret);
@@ -53,18 +130,51 @@ export function withRedactedOutput(deps, argv) {
53
130
  // A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
54
131
  // 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
55
132
  encoded.add(encodeURIComponent(`${secret}@`));
133
+ // What a server echoes back: the Basic value and the decoded user:password on
134
+ // stdout and stderr, the password alone (it may well occur in the data) on stderr.
135
+ const [basic, pair, password] = echoedCredentialForms(secret);
136
+ if (basic !== undefined)
137
+ echoed.add(basic);
138
+ if (pair !== undefined)
139
+ echoed.add(pair);
140
+ if (password !== undefined)
141
+ passwords.add(password);
56
142
  }
57
143
  }
58
144
  if (secrets.size === 0)
59
- return deps;
145
+ return { out: (text) => text, err: (text) => text };
60
146
  const list = [...secrets];
61
- const redact = (text) => {
62
- let out = redactUserinfo(redactCredentials(text, list));
147
+ // Longest first, so a secret is never left half-replaced by one of its own substrings.
148
+ const echoedList = [...echoed].sort((a, b) => b.length - a.length);
149
+ const passwordList = [...passwords].sort((a, b) => b.length - a.length);
150
+ const out = (text) => {
151
+ let result = redactUserinfo(redactCredentials(text, list));
63
152
  for (const form of encoded)
64
- out = out.split(form).join("***%40");
65
- return out;
153
+ result = result.split(form).join("***%40");
154
+ return redactSecrets(result, echoedList);
155
+ };
156
+ return { out, err: (text) => redactSecrets(out(text), passwordList) };
157
+ }
158
+ /**
159
+ * `deps` that keep the secrets of this run (`redactionFor`) out of everything they
160
+ * print: `io.out` is redacted, and the log (`deps.log`) replaces them in each record's
161
+ * message before formatting it, then writes to the raw `io.err`, so the frame is never
162
+ * touched and a password with DEL, C1 or bidi characters is matched in its raw form.
163
+ * `io.err` itself is redacted too, for anything that writes to stderr without the log.
164
+ */
165
+ export function withRedactedOutput(deps, argv) {
166
+ const redaction = redactionFor(argv, deps.env);
167
+ const { out, err } = deps.io;
168
+ return {
169
+ ...deps,
170
+ io: { ...deps.io, out: (text) => out(redaction.out(text)), err: (text) => err(redaction.err(text)) },
171
+ log: createLogger({
172
+ format: logFormatFromArgv(argv),
173
+ write: err,
174
+ redact: redaction.err,
175
+ ...(deps.now === undefined ? {} : { now: deps.now }),
176
+ }),
66
177
  };
67
- return { ...deps, io: { out: (text) => deps.io.out(redact(text)), err: (text) => deps.io.err(redact(text)) } };
68
178
  }
69
179
  /**
70
180
  * The index of a `help` command whose topic names no command (`ckan help nonexistent`),
@@ -95,10 +205,44 @@ export function unknownHelpTopic(program, argv) {
95
205
  }
96
206
  return undefined;
97
207
  }
208
+ /**
209
+ * The log for what happens outside `run()`, in the bin shim: a stdout write error
210
+ * (`handleOutputErrors`) and Node's process warnings. Its format is the one argv asks
211
+ * for (`logFormatFromArgv`), and it replaces the secrets of argv and `env` like the
212
+ * run's own log; it writes to the raw stderr.
213
+ */
214
+ export function processLogger(argv, env = process.env) {
215
+ return createLogger({
216
+ format: logFormatFromArgv(argv),
217
+ write: (line) => process.stderr.write(line + "\n"),
218
+ redact: redactionFor(argv, env).err,
219
+ });
220
+ }
221
+ /**
222
+ * The log area of a `CkanError` that is neither an API error nor a usage error: the
223
+ * connection (`http`), a malformed answer (`api`: bad JSON, not JSON, not a CKAN
224
+ * envelope, the wrong result shape, an unknown charset — the portal's answer as much as
225
+ * an error status is), CKAN's own error answer on HTTP 200 (`api`: a `success: false`
226
+ * envelope, `CkanActionError`), else `cli` (a response nested too deeply to print is
227
+ * about printing the answer, not its shape).
228
+ */
229
+ function areaOf(err) {
230
+ if (err instanceof CkanNetworkError)
231
+ return "http";
232
+ if (err instanceof CkanParseError || err instanceof CkanActionError)
233
+ return "api";
234
+ return "cli";
235
+ }
98
236
  export async function run(argv, deps = defaultDeps) {
237
+ // The log replaces the secrets of the run in every message, in either format.
99
238
  deps = withRedactedOutput(deps, argv);
100
239
  const program = buildProgram(deps);
101
240
  configureTree(program, deps);
241
+ // For the records of a parse error: the scan of argv, now knowing which of the program's
242
+ // options take a value, as commander reads them. Once commander has parsed argv, the program's
243
+ // first preAction hook sets the format it parsed (`buildProgram`).
244
+ if (deps.log !== undefined)
245
+ deps.log.format = logFormatFromArgv(argv, valueOptionsOf(program));
102
246
  // `help <unknown>` is answered like `<unknown>`: "error: unknown command '<unknown>'",
103
247
  // the help after it, exit 1 — not the help alone with a failure code.
104
248
  const helpAt = unknownHelpTopic(program, argv);
@@ -113,24 +257,30 @@ export async function run(argv, deps = defaultDeps) {
113
257
  // Help/version requests exit 0; genuine parse errors carry their own code.
114
258
  return err.exitCode;
115
259
  }
260
+ const log = logOf(deps);
116
261
  if (err instanceof CkanValidationError) {
117
262
  // An input the library refused before any request (a rule the commander
118
263
  // parsers do not see, e.g. one across parameters): a usage error.
119
- deps.io.err(`Error: ${err.message}`);
264
+ log.error("cli", err.message);
120
265
  return USAGE_EXIT;
121
266
  }
122
267
  if (err instanceof CkanApiError) {
123
- deps.io.err(`Error: ${err.message}`);
268
+ log.error("api", err.message);
124
269
  // Map a few notable statuses to distinct exit codes for scripting.
125
270
  if (err.status === 404)
126
271
  return 4;
127
272
  return 1;
128
273
  }
274
+ if (err instanceof CkanActionError) {
275
+ log.error(areaOf(err), err.message);
276
+ // CKAN's Not Found Error is a not-found whatever the HTTP status it came with.
277
+ return err.isNotFound ? 4 : 1;
278
+ }
129
279
  if (err instanceof CkanError) {
130
- deps.io.err(`Error: ${err.message}`);
280
+ log.error(areaOf(err), err.message);
131
281
  return 1;
132
282
  }
133
- deps.io.err(`Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
283
+ log.error("cli", `Unexpected error: ${err instanceof Error ? err.message : String(err)}`);
134
284
  return 1;
135
285
  }
136
286
  }
@@ -1,5 +1,5 @@
1
- import type { CliDeps } from "./io.js";
2
- import { type EngineOptions } from "../client/engine.js";
1
+ import { type CliDeps } from "./io.js";
2
+ import { type EngineOptions, type RetryEvent } from "../client/engine.js";
3
3
  /** commander value-parser: a non-negative integer. */
4
4
  export declare function parseIntArg(value: string): number;
5
5
  /**
@@ -66,6 +66,16 @@ export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
66
66
  export declare function escapeControlChars(json: string): string;
67
67
  /** Render a JSON value to stdout, pretty by default, compact with --compact. */
68
68
  export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
69
+ /**
70
+ * Log a warning (`ckan.http`) when the effective base URL (--portal >
71
+ * --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
72
+ * (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
73
+ * Called once per run, after the options are parsed and before the first request;
74
+ * stdout and the exit code are untouched.
75
+ */
76
+ export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
77
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
78
+ export declare function retryMessage(event: RetryEvent): string;
69
79
  export interface ActionContext {
70
80
  client: ReturnType<CliDeps["createClient"]>;
71
81
  global: GlobalOptions;