@maschinenlesbar.org/ckan-cli 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +25 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +63 -22
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.d.ts +441 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.js +1211 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +2 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.d.ts +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.js +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.d.ts +80 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.js +124 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/package.json +25 -0
- package/dist/src/cli/index.js +3 -1
- package/dist/src/cli/io.d.ts +17 -0
- package/dist/src/cli/io.js +14 -0
- package/dist/src/cli/limits.d.ts +19 -0
- package/dist/src/cli/limits.js +53 -0
- package/dist/src/cli/program.js +3 -0
- package/dist/src/cli/run.js +9 -1
- package/dist/src/cli/shared.d.ts +4 -0
- package/dist/src/cli/shared.js +16 -1
- package/node_modules/commander/Readme.md +6 -32
- package/node_modules/commander/index.js +1 -1
- package/node_modules/commander/lib/command.js +22 -9
- package/node_modules/commander/lib/help.js +6 -2
- package/node_modules/commander/lib/option.js +1 -1
- package/node_modules/commander/package.json +2 -1
- package/node_modules/commander/typings/index.d.ts +6 -3
- package/package.json +2 -2
|
@@ -30,6 +30,8 @@ export interface PortalCheck {
|
|
|
30
30
|
* Never throws: a failure is described in `problem`.
|
|
31
31
|
*/
|
|
32
32
|
export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
|
|
33
|
+
/** A short, stable reason for a failed check (or a failed request to a portal). */
|
|
34
|
+
export declare function problemOf(err: unknown): string;
|
|
33
35
|
/**
|
|
34
36
|
* A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
|
|
35
37
|
* check keeps the last known version and dataset count, so an entry that is down
|
|
@@ -87,8 +87,8 @@ function errorCode(cause, depth = 0) {
|
|
|
87
87
|
const code = cause.code;
|
|
88
88
|
return code !== undefined ? code : errorCode(cause.cause, depth + 1);
|
|
89
89
|
}
|
|
90
|
-
/** A short, stable reason for a failed check. */
|
|
91
|
-
function problemOf(err) {
|
|
90
|
+
/** A short, stable reason for a failed check (or a failed request to a portal). */
|
|
91
|
+
export function problemOf(err) {
|
|
92
92
|
if (err instanceof CkanApiError) {
|
|
93
93
|
return err.detail?.startsWith("stopped after") ? "redirect loop" : `HTTP ${err.status}`;
|
|
94
94
|
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./log.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from "./log.js";
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/** The log formats `--log-format` takes. */
|
|
2
|
+
export declare const LOG_FORMATS: readonly ["text", "jsonl"];
|
|
3
|
+
export type LogFormat = (typeof LOG_FORMATS)[number];
|
|
4
|
+
/** The format without `--log-format`. */
|
|
5
|
+
export declare const DEFAULT_LOG_FORMAT: LogFormat;
|
|
6
|
+
type LogLevel = "ERROR" | "WARN" | "INFO";
|
|
7
|
+
interface LogRecord {
|
|
8
|
+
/** ISO 8601, UTC, milliseconds. */
|
|
9
|
+
ts: string;
|
|
10
|
+
level: LogLevel;
|
|
11
|
+
/** `<program>.<area>`, like `ckan.http`. */
|
|
12
|
+
topic: string;
|
|
13
|
+
msg: string;
|
|
14
|
+
}
|
|
15
|
+
/**
|
|
16
|
+
* `text` with every character that could split a record, forge a second one or steer
|
|
17
|
+
* the terminal written as an escape: CR as `\r`, LF as `\n`, any other C0 control but
|
|
18
|
+
* TAB, DEL and C1 as `\u00XX`, U+2028, U+2029 and the bidi controls (U+061C, U+200E,
|
|
19
|
+
* U+200F, U+202A–U+202E, U+2066–U+2069) as `\uXXXX`. Backslashes stay as they are. On
|
|
20
|
+
* the output of `JSON.stringify` (no raw C0 left) every escape it adds is valid JSON,
|
|
21
|
+
* so the same helper serves both formats. Checked by char code, so the source stays
|
|
22
|
+
* free of those characters.
|
|
23
|
+
*/
|
|
24
|
+
export declare function escapeForRecord(text: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* The longest message (in characters) a record carries. A longer one is cut at a
|
|
27
|
+
* code-point boundary and ends in `… (N more characters)`: one huge server text or typed
|
|
28
|
+
* value cannot flood stderr or a log store with a line of megabytes. Own messages that
|
|
29
|
+
* quote server or user data are bounded at their source too (`cutForMessage`); this
|
|
30
|
+
* is the backstop for every path.
|
|
31
|
+
*/
|
|
32
|
+
export declare const MAX_RECORD_MESSAGE = 4000;
|
|
33
|
+
/**
|
|
34
|
+
* One record as one line, whatever the message holds: `escapeForRecord` runs over the
|
|
35
|
+
* message (text) or over the whole JSON object (jsonl), so no text that reaches a
|
|
36
|
+
* record can split it, forge another one, or reach the terminal as a control sequence.
|
|
37
|
+
*/
|
|
38
|
+
export declare function formatLogRecord(record: LogRecord, format: LogFormat): string;
|
|
39
|
+
export interface Logger {
|
|
40
|
+
/**
|
|
41
|
+
* The format records are written in. Mutable for one reason: it is read from argv
|
|
42
|
+
* before commander parses (for commander's own errors), then set again from what
|
|
43
|
+
* commander parsed, before the action runs (the CLI program's first preAction hook).
|
|
44
|
+
*/
|
|
45
|
+
format: LogFormat;
|
|
46
|
+
error(area: string, msg: string): void;
|
|
47
|
+
warn(area: string, msg: string): void;
|
|
48
|
+
info(area: string, msg: string): void;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* A logger that writes each record, formatted, to `write` (stderr). `redact` replaces the
|
|
52
|
+
* secrets of the run in the message, before it is cut and escaped: the frame (time,
|
|
53
|
+
* level, topic) is never touched, so a secret equal to a year or a topic cannot corrupt
|
|
54
|
+
* it, and a secret holding DEL, C1 or bidi characters is found in its raw form.
|
|
55
|
+
*/
|
|
56
|
+
export declare function createLogger(options: {
|
|
57
|
+
/** The first part of every topic: `<program>.<area>`. */
|
|
58
|
+
program: string;
|
|
59
|
+
format: LogFormat;
|
|
60
|
+
write: (line: string) => void;
|
|
61
|
+
now?: () => Date;
|
|
62
|
+
redact?: (text: string) => string;
|
|
63
|
+
}): Logger;
|
|
64
|
+
/** Why `value` is not a log format, or undefined. */
|
|
65
|
+
export declare function logFormatProblem(value: string): string | undefined;
|
|
66
|
+
/** What `installWarningLog` needs of the process. */
|
|
67
|
+
interface WarningSource {
|
|
68
|
+
removeAllListeners(event: "warning"): unknown;
|
|
69
|
+
on(event: "warning", listener: (warning: Error) => void): unknown;
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Node's own process warnings (`(node:PID) Warning: …`, e.g. for
|
|
73
|
+
* `NODE_TLS_REJECT_UNAUTHORIZED=0`) as WARN records of `area` (the CLI's `cli`):
|
|
74
|
+
* `(node) <name>: <message>`, in the log's format and with its redaction. Node's default
|
|
75
|
+
* listener, which prints the plain line, is removed (checked on Node 22, 24 and 26: that
|
|
76
|
+
* silences it; `--no-warnings` is not needed). The bin shim installs it once, before
|
|
77
|
+
* `run()`; a warning Node emits while it starts up, before any code runs, stays Node's.
|
|
78
|
+
*/
|
|
79
|
+
export declare function installWarningLog(source: WarningSource, log: Pick<Logger, "warn">, area?: string): void;
|
|
80
|
+
export {};
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// The apps' log (CLI and MCP server): every diagnostic line on stderr — errors, warnings,
|
|
2
|
+
// notes — is one record with a timestamp, a level and a topic. Two formats, chosen with the
|
|
3
|
+
// apps' `--log-format`:
|
|
4
|
+
//
|
|
5
|
+
// text 2026-10-09T14:03:12.481Z WARN [ckan.http] requests to … are sent unencrypted …
|
|
6
|
+
// jsonl {"ts":"2026-10-09T14:03:12.481Z","level":"WARN","topic":"ckan.http","msg":"…"}
|
|
7
|
+
//
|
|
8
|
+
// The text form follows log4j's pattern (`%d %-5p [%c] %m`), with the time in UTC
|
|
9
|
+
// ISO 8601. The topic is a dotted logger name: the program, then the area the record
|
|
10
|
+
// comes from (`ckan.cli`, `ckan.api`, `ckan.http`;
|
|
11
|
+
// `ckan-mcp.mcp` for the MCP server). Each app binds its program name
|
|
12
|
+
// (`createLogger({ program })`).
|
|
13
|
+
// stdout carries data only and is not touched; neither is `--help`/`--version`.
|
|
14
|
+
import { cutText, toWellFormed } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
15
|
+
/** The log formats `--log-format` takes. */
|
|
16
|
+
export const LOG_FORMATS = ["text", "jsonl"];
|
|
17
|
+
/** The format without `--log-format`. */
|
|
18
|
+
export const DEFAULT_LOG_FORMAT = "text";
|
|
19
|
+
/** True for a character a record never carries raw (see `escapeForRecord`). */
|
|
20
|
+
function escapedInRecords(c) {
|
|
21
|
+
return ((c < 0x20 && c !== 0x09) || // C0 but TAB
|
|
22
|
+
(c >= 0x7f && c <= 0x9f) || // DEL and C1 (NEL, the 8-bit CSI)
|
|
23
|
+
c === 0x2028 || c === 0x2029 || // line and paragraph separator
|
|
24
|
+
c === 0x061c || c === 0x200e || c === 0x200f || // bidi marks
|
|
25
|
+
(c >= 0x202a && c <= 0x202e) || // bidi embeddings and overrides
|
|
26
|
+
(c >= 0x2066 && c <= 0x2069) // bidi isolates
|
|
27
|
+
);
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* `text` with every character that could split a record, forge a second one or steer
|
|
31
|
+
* the terminal written as an escape: CR as `\r`, LF as `\n`, any other C0 control but
|
|
32
|
+
* TAB, DEL and C1 as `\u00XX`, U+2028, U+2029 and the bidi controls (U+061C, U+200E,
|
|
33
|
+
* U+200F, U+202A–U+202E, U+2066–U+2069) as `\uXXXX`. Backslashes stay as they are. On
|
|
34
|
+
* the output of `JSON.stringify` (no raw C0 left) every escape it adds is valid JSON,
|
|
35
|
+
* so the same helper serves both formats. Checked by char code, so the source stays
|
|
36
|
+
* free of those characters.
|
|
37
|
+
*/
|
|
38
|
+
export function escapeForRecord(text) {
|
|
39
|
+
let out = "";
|
|
40
|
+
let from = 0;
|
|
41
|
+
for (let i = 0; i < text.length; i++) {
|
|
42
|
+
const c = text.charCodeAt(i);
|
|
43
|
+
if (!escapedInRecords(c))
|
|
44
|
+
continue;
|
|
45
|
+
const escaped = c === 0x0d ? "\\r" : c === 0x0a ? "\\n" : "\\u" + c.toString(16).padStart(4, "0");
|
|
46
|
+
out += text.slice(from, i) + escaped;
|
|
47
|
+
from = i + 1;
|
|
48
|
+
}
|
|
49
|
+
return from === 0 ? text : out + text.slice(from);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The longest message (in characters) a record carries. A longer one is cut at a
|
|
53
|
+
* code-point boundary and ends in `… (N more characters)`: one huge server text or typed
|
|
54
|
+
* value cannot flood stderr or a log store with a line of megabytes. Own messages that
|
|
55
|
+
* quote server or user data are bounded at their source too (`cutForMessage`); this
|
|
56
|
+
* is the backstop for every path.
|
|
57
|
+
*/
|
|
58
|
+
export const MAX_RECORD_MESSAGE = 4000;
|
|
59
|
+
/** `msg` within `MAX_RECORD_MESSAGE`, the cut marked with the number of characters left out. */
|
|
60
|
+
function boundRecordMessage(msg) {
|
|
61
|
+
if (msg.length <= MAX_RECORD_MESSAGE)
|
|
62
|
+
return msg;
|
|
63
|
+
const kept = cutText(msg, MAX_RECORD_MESSAGE);
|
|
64
|
+
return `${kept}… (${codePoints(msg, kept.length)} more characters)`;
|
|
65
|
+
}
|
|
66
|
+
/** The number of characters (code points) in `text` from `from` on. */
|
|
67
|
+
function codePoints(text, from) {
|
|
68
|
+
let n = 0;
|
|
69
|
+
for (let i = from; i < text.length; i++) {
|
|
70
|
+
const c = text.charCodeAt(i);
|
|
71
|
+
// The low half of a pair is not a character of its own.
|
|
72
|
+
if (c >= 0xdc00 && c <= 0xdfff && i > from && text.charCodeAt(i - 1) >= 0xd800 && text.charCodeAt(i - 1) <= 0xdbff)
|
|
73
|
+
continue;
|
|
74
|
+
n++;
|
|
75
|
+
}
|
|
76
|
+
return n;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* One record as one line, whatever the message holds: `escapeForRecord` runs over the
|
|
80
|
+
* message (text) or over the whole JSON object (jsonl), so no text that reaches a
|
|
81
|
+
* record can split it, forge another one, or reach the terminal as a control sequence.
|
|
82
|
+
*/
|
|
83
|
+
export function formatLogRecord(record, format) {
|
|
84
|
+
// Well-formed first: half a character would be `\ud83d` in jsonl, which jq rejects,
|
|
85
|
+
// stopping the whole stream.
|
|
86
|
+
const msg = toWellFormed(boundRecordMessage(record.msg));
|
|
87
|
+
if (format === "jsonl")
|
|
88
|
+
return escapeForRecord(JSON.stringify({ ts: record.ts, level: record.level, topic: record.topic, msg }));
|
|
89
|
+
return `${record.ts} ${record.level.padEnd(5)} [${record.topic}] ${escapeForRecord(msg)}`;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* A logger that writes each record, formatted, to `write` (stderr). `redact` replaces the
|
|
93
|
+
* secrets of the run in the message, before it is cut and escaped: the frame (time,
|
|
94
|
+
* level, topic) is never touched, so a secret equal to a year or a topic cannot corrupt
|
|
95
|
+
* it, and a secret holding DEL, C1 or bidi characters is found in its raw form.
|
|
96
|
+
*/
|
|
97
|
+
export function createLogger(options) {
|
|
98
|
+
const now = options.now ?? (() => new Date());
|
|
99
|
+
const redact = options.redact ?? ((text) => text);
|
|
100
|
+
const logger = {
|
|
101
|
+
format: options.format,
|
|
102
|
+
error: (area, msg) => log("ERROR", area, msg),
|
|
103
|
+
warn: (area, msg) => log("WARN", area, msg),
|
|
104
|
+
info: (area, msg) => log("INFO", area, msg),
|
|
105
|
+
};
|
|
106
|
+
const log = (level, area, msg) => options.write(formatLogRecord({ ts: now().toISOString(), level, topic: `${options.program}.${area}`, msg: redact(msg) }, logger.format));
|
|
107
|
+
return logger;
|
|
108
|
+
}
|
|
109
|
+
/** Why `value` is not a log format, or undefined. */
|
|
110
|
+
export function logFormatProblem(value) {
|
|
111
|
+
return LOG_FORMATS.includes(value) ? undefined : `Expected one of ${LOG_FORMATS.join(", ")}.`;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Node's own process warnings (`(node:PID) Warning: …`, e.g. for
|
|
115
|
+
* `NODE_TLS_REJECT_UNAUTHORIZED=0`) as WARN records of `area` (the CLI's `cli`):
|
|
116
|
+
* `(node) <name>: <message>`, in the log's format and with its redaction. Node's default
|
|
117
|
+
* listener, which prints the plain line, is removed (checked on Node 22, 24 and 26: that
|
|
118
|
+
* silences it; `--no-warnings` is not needed). The bin shim installs it once, before
|
|
119
|
+
* `run()`; a warning Node emits while it starts up, before any code runs, stays Node's.
|
|
120
|
+
*/
|
|
121
|
+
export function installWarningLog(source, log, area = "cli") {
|
|
122
|
+
source.removeAllListeners("warning");
|
|
123
|
+
source.on("warning", (warning) => log.warn(area, `(node) ${warning.name}: ${warning.message}`));
|
|
124
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@maschinenlesbar.org/ckan-log",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"private": true,
|
|
5
|
+
"description": "The log records the apps write on stderr (text or jsonl), shared by the CLI and the MCP server.",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": {
|
|
9
|
+
"types": "./dist/src/index.d.ts",
|
|
10
|
+
"import": "./dist/src/index.js"
|
|
11
|
+
},
|
|
12
|
+
"./package.json": "./package.json"
|
|
13
|
+
},
|
|
14
|
+
"files": [
|
|
15
|
+
"dist/src"
|
|
16
|
+
],
|
|
17
|
+
"license": "AGPL-3.0-or-later",
|
|
18
|
+
"author": "Sebastian Schürmann <sebs@2xs.org>",
|
|
19
|
+
"repository": {
|
|
20
|
+
"type": "git",
|
|
21
|
+
"url": "git+https://github.com/maschinenlesbar-org/ckan-cli.git",
|
|
22
|
+
"directory": "packages/log"
|
|
23
|
+
},
|
|
24
|
+
"homepage": "https://maschinenlesbar.org/ckan-cli/"
|
|
25
|
+
}
|
package/dist/src/cli/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
// Binary entry point. Thin shim around run(); all logic lives in run.ts/program.ts.
|
|
3
|
-
import { handleOutputErrors } from "./io.js";
|
|
3
|
+
import { exitOnSignalsAsPid1, handleOutputErrors } from "./io.js";
|
|
4
4
|
import { installWarningLog } from "./log.js";
|
|
5
5
|
import { processLogger, run } from "./run.js";
|
|
6
6
|
const argv = process.argv.slice(2);
|
|
@@ -8,5 +8,7 @@ const argv = process.argv.slice(2);
|
|
|
8
8
|
const log = processLogger(argv);
|
|
9
9
|
installWarningLog(process, log);
|
|
10
10
|
handleOutputErrors(process, undefined, log);
|
|
11
|
+
// In a container the CLI is PID 1, which ignores a signal nothing listens for.
|
|
12
|
+
exitOnSignalsAsPid1();
|
|
11
13
|
const exitCode = await run(argv);
|
|
12
14
|
process.exitCode = exitCode;
|
package/dist/src/cli/io.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { CkanClient } from "@maschinenlesbar.org/ckan-client/internal/client";
|
|
2
2
|
import type { EngineOptions } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
3
3
|
import { type Logger } from "./log.js";
|
|
4
|
+
import type { ContainerLimits } from "./limits.js";
|
|
4
5
|
export interface CliIO {
|
|
5
6
|
out(text: string): void;
|
|
6
7
|
err(text: string): void;
|
|
@@ -18,6 +19,8 @@ export interface CliDeps {
|
|
|
18
19
|
log?: Logger;
|
|
19
20
|
/** The clock the log's timestamps come from. Unset, the real one. */
|
|
20
21
|
now?: () => Date;
|
|
22
|
+
/** The container's limits (limits.ts); only the bin's defaultDeps reads them, never a test. */
|
|
23
|
+
limits?: () => ContainerLimits;
|
|
21
24
|
}
|
|
22
25
|
/** The deps' logger, or one that writes text records through `io.err`. */
|
|
23
26
|
export declare function logOf(deps: CliDeps): Logger;
|
|
@@ -74,4 +77,18 @@ interface Stdout {
|
|
|
74
77
|
*/
|
|
75
78
|
export declare function stdoutWriter(stdout?: Stdout, write?: (fd: number, data: Buffer) => void): (text: string) => void;
|
|
76
79
|
export declare const defaultIO: CliIO;
|
|
80
|
+
/** What `exitOnSignalsAsPid1` needs of the process. */
|
|
81
|
+
export interface SignalProcess {
|
|
82
|
+
readonly pid: number;
|
|
83
|
+
once(event: string, listener: () => void): unknown;
|
|
84
|
+
exit(code: number): void;
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* As PID 1 (the first process of a container), a signal without a listener is ignored: Node's
|
|
88
|
+
* default action re-raises it, and the kernel drops a default-action signal sent to PID 1, so
|
|
89
|
+
* `docker stop` waited 10 s and then killed the run. There, and only there, the bin shim listens
|
|
90
|
+
* for SIGINT, SIGTERM and SIGHUP from the start and exits with 130, 143 or 129. Anywhere else
|
|
91
|
+
* Node's default action stays.
|
|
92
|
+
*/
|
|
93
|
+
export declare function exitOnSignalsAsPid1(proc?: SignalProcess): void;
|
|
77
94
|
export {};
|
package/dist/src/cli/io.js
CHANGED
|
@@ -137,3 +137,17 @@ export const defaultIO = {
|
|
|
137
137
|
// A record never lands inside the data when both streams share a pipe.
|
|
138
138
|
err: stderrLine,
|
|
139
139
|
};
|
|
140
|
+
/**
|
|
141
|
+
* As PID 1 (the first process of a container), a signal without a listener is ignored: Node's
|
|
142
|
+
* default action re-raises it, and the kernel drops a default-action signal sent to PID 1, so
|
|
143
|
+
* `docker stop` waited 10 s and then killed the run. There, and only there, the bin shim listens
|
|
144
|
+
* for SIGINT, SIGTERM and SIGHUP from the start and exits with 130, 143 or 129. Anywhere else
|
|
145
|
+
* Node's default action stays.
|
|
146
|
+
*/
|
|
147
|
+
export function exitOnSignalsAsPid1(proc = process) {
|
|
148
|
+
if (proc.pid !== 1)
|
|
149
|
+
return;
|
|
150
|
+
for (const [signal, code] of [["SIGINT", 130], ["SIGTERM", 143], ["SIGHUP", 129]]) {
|
|
151
|
+
proc.once(signal, () => proc.exit(code));
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Why the CLI can't start a request under these limits, or undefined: a pids limit Node can't
|
|
3
|
+
* work with. The CLI's `action()` raises it as an error of the cli area (exit 1) before any
|
|
4
|
+
* request.
|
|
5
|
+
*/
|
|
6
|
+
export declare function pidsProblem(limits: ContainerLimits): string | undefined;
|
|
7
|
+
export interface ContainerLimits {
|
|
8
|
+
/** The memory limit in bytes, when there is one. */
|
|
9
|
+
memoryBytes?: number;
|
|
10
|
+
/** The most processes and threads, when there is a limit. */
|
|
11
|
+
pids?: number;
|
|
12
|
+
}
|
|
13
|
+
/** The limits in the cgroup files, read with `read` (default: the file system). */
|
|
14
|
+
export declare function readContainerLimits(read?: (path: string) => string): ContainerLimits;
|
|
15
|
+
/**
|
|
16
|
+
* The default response cap under a memory limit: an eighth of it (decoding and parsing an answer
|
|
17
|
+
* takes several times its size), at least 1 MiB, at most the client's default.
|
|
18
|
+
*/
|
|
19
|
+
export declare function responseCapFor(memoryBytes: number): number;
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
// The container's limits, read from its cgroup (v2, then v1), for what the CLI can't otherwise
|
|
2
|
+
// see: in a memory-limited container, a response cap that the memory can't hold turns an
|
|
3
|
+
// oversized answer into the kernel's OOM kill (exit 137, nothing said) instead of the cap's error.
|
|
4
|
+
// Outside a container (or on macOS) nothing is found and nothing changes.
|
|
5
|
+
import { readFileSync } from "node:fs";
|
|
6
|
+
/**
|
|
7
|
+
* The fewest processes and threads Node needs for a request (its libuv pool and V8's platform
|
|
8
|
+
* threads): under 12 it died with exit 133 (8) or hung (4, before any code of the CLI runs).
|
|
9
|
+
*/
|
|
10
|
+
const MIN_PIDS = 12;
|
|
11
|
+
/**
|
|
12
|
+
* Why the CLI can't start a request under these limits, or undefined: a pids limit Node can't
|
|
13
|
+
* work with. The CLI's `action()` raises it as an error of the cli area (exit 1) before any
|
|
14
|
+
* request.
|
|
15
|
+
*/
|
|
16
|
+
export function pidsProblem(limits) {
|
|
17
|
+
if (limits.pids === undefined || limits.pids >= MIN_PIDS)
|
|
18
|
+
return undefined;
|
|
19
|
+
return `the container allows only ${limits.pids} processes and threads (--pids-limit ${limits.pids}); the CLI needs at least ${MIN_PIDS}. Run it with --pids-limit ${MIN_PIDS} or more.`;
|
|
20
|
+
}
|
|
21
|
+
/** The client's default response cap (100 MiB), the most a memory-derived cap gives. */
|
|
22
|
+
const DEFAULT_RESPONSE_CAP = 100 * 1024 * 1024;
|
|
23
|
+
// cgroup v1 writes "no limit" as a huge number (about 2^63) instead of "max".
|
|
24
|
+
const UNLIMITED = 2 ** 50;
|
|
25
|
+
/** The limits in the cgroup files, read with `read` (default: the file system). */
|
|
26
|
+
export function readContainerLimits(read = (path) => readFileSync(path, "utf8")) {
|
|
27
|
+
const first = (...paths) => {
|
|
28
|
+
for (const path of paths) {
|
|
29
|
+
let text;
|
|
30
|
+
try {
|
|
31
|
+
text = read(path).trim();
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (!/^\d+$/.test(text))
|
|
37
|
+
return undefined; // "max": no limit
|
|
38
|
+
const value = Number(text);
|
|
39
|
+
return Number.isSafeInteger(value) && value < UNLIMITED ? value : undefined;
|
|
40
|
+
}
|
|
41
|
+
return undefined;
|
|
42
|
+
};
|
|
43
|
+
const memoryBytes = first("/sys/fs/cgroup/memory.max", "/sys/fs/cgroup/memory/memory.limit_in_bytes");
|
|
44
|
+
const pids = first("/sys/fs/cgroup/pids.max", "/sys/fs/cgroup/pids/pids.max");
|
|
45
|
+
return { ...(memoryBytes === undefined ? {} : { memoryBytes }), ...(pids === undefined ? {} : { pids }) };
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* The default response cap under a memory limit: an eighth of it (decoding and parsing an answer
|
|
49
|
+
* takes several times its size), at least 1 MiB, at most the client's default.
|
|
50
|
+
*/
|
|
51
|
+
export function responseCapFor(memoryBytes) {
|
|
52
|
+
return Math.min(DEFAULT_RESPONSE_CAP, Math.max(1024 * 1024, Math.floor(memoryBytes / 8)));
|
|
53
|
+
}
|
package/dist/src/cli/program.js
CHANGED
|
@@ -5,6 +5,7 @@ import { readFileSync } from "node:fs";
|
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { Command, InvalidArgumentError, Option } from "commander";
|
|
7
7
|
import { defaultIO } from "./io.js";
|
|
8
|
+
import { readContainerLimits } from "./limits.js";
|
|
8
9
|
import { CkanClient } from "@maschinenlesbar.org/ckan-client/internal/client";
|
|
9
10
|
import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
10
11
|
import { MAX_TIMEOUT_MS } from "@maschinenlesbar.org/ckan-client/internal/http";
|
|
@@ -35,6 +36,8 @@ export const defaultDeps = {
|
|
|
35
36
|
io: defaultIO,
|
|
36
37
|
env: process.env,
|
|
37
38
|
createClient: (options) => new CkanClient(options),
|
|
39
|
+
// In a container: the memory and pids limits (cgroup), for action()'s checks.
|
|
40
|
+
limits: () => readContainerLimits(),
|
|
38
41
|
};
|
|
39
42
|
/** commander value-parser for `--log-format`. */
|
|
40
43
|
function parseLogFormat(value) {
|
package/dist/src/cli/run.js
CHANGED
|
@@ -5,7 +5,8 @@ import { CommanderError } from "commander";
|
|
|
5
5
|
import { buildProgram, defaultDeps } from "./program.js";
|
|
6
6
|
import { logOf } from "./io.js";
|
|
7
7
|
import { OutputError } from "./output.js";
|
|
8
|
-
import { createLogger, logFormatFromArgv } from "./log.js";
|
|
8
|
+
import { createLogger, LOG_PROGRAM, logFormatFromArgv } from "./log.js";
|
|
9
|
+
import { EnvironmentError } from "./shared.js";
|
|
9
10
|
import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, redactUrlUserinfo, typedUrlCredentials, } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
10
11
|
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
11
12
|
const USAGE_EXIT = 1;
|
|
@@ -290,6 +291,8 @@ export function processLogger(argv, env = process.env) {
|
|
|
290
291
|
function areaOf(err) {
|
|
291
292
|
if (err instanceof OutputError)
|
|
292
293
|
return "output";
|
|
294
|
+
if (err instanceof EnvironmentError)
|
|
295
|
+
return "cli";
|
|
293
296
|
if (err instanceof CkanNetworkError)
|
|
294
297
|
return "http";
|
|
295
298
|
if (err instanceof CkanParseError || err instanceof CkanActionError)
|
|
@@ -297,6 +300,11 @@ function areaOf(err) {
|
|
|
297
300
|
return "cli";
|
|
298
301
|
}
|
|
299
302
|
export async function run(argv, deps = defaultDeps) {
|
|
303
|
+
// In the container image the entrypoint is the program, so `docker run <image> search Radverkehr`;
|
|
304
|
+
// people copy the npm form and type `docker run <image> ckan search Radverkehr`. A leading
|
|
305
|
+
// program name is dropped, so both work (no command is named like the program).
|
|
306
|
+
if (argv[0] === LOG_PROGRAM)
|
|
307
|
+
argv = argv.slice(1);
|
|
300
308
|
// The log replaces the secrets of the run in every message, in either format.
|
|
301
309
|
deps = withRedactedOutput(deps, argv);
|
|
302
310
|
const program = buildProgram(deps);
|
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type CliDeps } from "./io.js";
|
|
2
2
|
import { type EngineOptions, type RetryEvent } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
3
|
+
import { CkanError } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
3
4
|
import { type Problem } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
4
5
|
/** commander value-parser: a non-negative integer. */
|
|
5
6
|
export declare function parseIntArg(value: string): number;
|
|
@@ -65,6 +66,9 @@ export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
|
|
|
65
66
|
export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
|
|
66
67
|
/** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
|
|
67
68
|
export declare function retryMessage(event: RetryEvent): string;
|
|
69
|
+
/** The container can't run the CLI as it is limited (limits.ts): an ERROR record of the cli area and exit 1. */
|
|
70
|
+
export declare class EnvironmentError extends CkanError {
|
|
71
|
+
}
|
|
68
72
|
export interface ActionContext {
|
|
69
73
|
client: ReturnType<CliDeps["createClient"]>;
|
|
70
74
|
global: GlobalOptions;
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
// option resolver. Results are written by output.ts.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
4
|
import { logOf } from "./io.js";
|
|
5
|
+
import { pidsProblem, responseCapFor } from "./limits.js";
|
|
5
6
|
import { cleartextProblem, DEFAULT_BASE_URL } from "@maschinenlesbar.org/ckan-client/internal/engine";
|
|
6
|
-
import { quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
7
|
+
import { CkanError, quoteValue, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
|
|
7
8
|
import { findPortal } from "@maschinenlesbar.org/ckan-client/internal/portals";
|
|
8
9
|
import { baseUrlProblem, countProblem, headerValueProblem, idProblem, intRangeProblem, textProblem, } from "@maschinenlesbar.org/ckan-client/internal/validate";
|
|
9
10
|
import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
|
|
@@ -161,6 +162,9 @@ export function retryMessage(event) {
|
|
|
161
162
|
const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
|
|
162
163
|
return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
|
|
163
164
|
}
|
|
165
|
+
/** The container can't run the CLI as it is limited (limits.ts): an ERROR record of the cli area and exit 1. */
|
|
166
|
+
export class EnvironmentError extends CkanError {
|
|
167
|
+
}
|
|
164
168
|
/**
|
|
165
169
|
* Wrap an async command action with consistent global-option resolution and
|
|
166
170
|
* client construction. The callback receives a context (client + resolved global
|
|
@@ -174,8 +178,19 @@ export function action(deps, fn) {
|
|
|
174
178
|
const command = args[args.length - 1];
|
|
175
179
|
const positionals = args.slice(0, Math.max(0, args.length - 2));
|
|
176
180
|
const global = command.optsWithGlobals();
|
|
181
|
+
// In a container: under a pids limit Node can't work with, refuse before any request; under a
|
|
182
|
+
// memory limit the 100 MiB default can't be held, so cap by the memory instead (an oversized
|
|
183
|
+
// answer is then the cap's error, not an OOM kill). An explicit --max-response-bytes wins.
|
|
184
|
+
const limits = deps.limits?.() ?? {};
|
|
185
|
+
const problem = pidsProblem(limits);
|
|
186
|
+
if (problem !== undefined)
|
|
187
|
+
throw new EnvironmentError(problem);
|
|
188
|
+
const given = command.getOptionValueSourceWithGlobals("maxResponseBytes");
|
|
189
|
+
const memory = given === undefined || given === "default" ? limits.memoryBytes : undefined;
|
|
177
190
|
warnOnCleartext(deps, global);
|
|
178
191
|
const options = toEngineOptions(global);
|
|
192
|
+
if (memory !== undefined)
|
|
193
|
+
options.maxResponseBytes = responseCapFor(memory);
|
|
179
194
|
options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
|
|
180
195
|
// An answer delivered changed (invalid text replaced with U+FFFD) is never silent.
|
|
181
196
|
options.onWarning = (message) => logOf(deps).warn("api", message);
|
|
@@ -44,7 +44,6 @@ Read this in other languages: English | [简体中文](./Readme_zh-CN.md)
|
|
|
44
44
|
- [Bits and pieces](#bits-and-pieces)
|
|
45
45
|
- [.parse() and .parseAsync()](#parse-and-parseasync)
|
|
46
46
|
- [Parsing Configuration](#parsing-configuration)
|
|
47
|
-
- [Legacy options as properties](#legacy-options-as-properties)
|
|
48
47
|
- [TypeScript](#typescript)
|
|
49
48
|
- [createCommand()](#createcommand)
|
|
50
49
|
- [Node options such as `--harmony`](#node-options-such-as---harmony)
|
|
@@ -54,7 +53,6 @@ Read this in other languages: English | [简体中文](./Readme_zh-CN.md)
|
|
|
54
53
|
- [Override exit and output handling](#override-exit-and-output-handling)
|
|
55
54
|
- [Additional documentation](#additional-documentation)
|
|
56
55
|
- [Support](#support)
|
|
57
|
-
- [Commander for enterprise](#commander-for-enterprise)
|
|
58
56
|
|
|
59
57
|
For information about terms used in this document see: [terminology](./docs/terminology.md)
|
|
60
58
|
|
|
@@ -367,7 +365,8 @@ Example file: [options-variadic.js](./examples/options-variadic.js)
|
|
|
367
365
|
```js
|
|
368
366
|
program
|
|
369
367
|
.option('-n, --number <numbers...>', 'specify numbers')
|
|
370
|
-
.option('-l, --letter [letters...]', 'specify letters')
|
|
368
|
+
.option('-l, --letter [letters...]', 'specify letters')
|
|
369
|
+
.argument('[operand]');
|
|
371
370
|
|
|
372
371
|
program.parse();
|
|
373
372
|
|
|
@@ -380,10 +379,10 @@ $ collect -n 1 2 3 --letter a b c
|
|
|
380
379
|
Options: { number: [ '1', '2', '3' ], letter: [ 'a', 'b', 'c' ] }
|
|
381
380
|
Remaining arguments: []
|
|
382
381
|
$ collect --letter=A -n80 operand
|
|
383
|
-
Options: {
|
|
382
|
+
Options: { letter: [ 'A' ], number: [ '80' ] }
|
|
384
383
|
Remaining arguments: [ 'operand' ]
|
|
385
384
|
$ collect --letter -n 1 -n 2 3 -- operand
|
|
386
|
-
Options: { number: [ '1', '2', '3' ]
|
|
385
|
+
Options: { letter: true, number: [ '1', '2', '3' ] }
|
|
387
386
|
Remaining arguments: [ 'operand' ]
|
|
388
387
|
```
|
|
389
388
|
|
|
@@ -670,7 +669,7 @@ Example file: [action-this.js](./examples/action-this.js)
|
|
|
670
669
|
program
|
|
671
670
|
.command('serve')
|
|
672
671
|
.argument('<script>')
|
|
673
|
-
.option('-p, --port <number>', 'port number', 80)
|
|
672
|
+
.option('-p, --port <number>', 'port number', '80')
|
|
674
673
|
.action(function() {
|
|
675
674
|
console.error('Run script %s on port %s', this.args[0], this.opts().port);
|
|
676
675
|
});
|
|
@@ -1016,25 +1015,6 @@ By default, the option processing shows an error for an unknown option. To have
|
|
|
1016
1015
|
By default, the argument processing displays an error for more command-arguments than expected.
|
|
1017
1016
|
To suppress the error for excess arguments, use `.allowExcessArguments()`.
|
|
1018
1017
|
|
|
1019
|
-
### Legacy options as properties
|
|
1020
|
-
|
|
1021
|
-
Before Commander 7, the option values were stored as properties on the command.
|
|
1022
|
-
This was convenient to code, but the downside was possible clashes with
|
|
1023
|
-
existing properties of `Command`.
|
|
1024
|
-
|
|
1025
|
-
You can revert to the old behaviour to run unmodified legacy code by using `.storeOptionsAsProperties()`.
|
|
1026
|
-
|
|
1027
|
-
```js
|
|
1028
|
-
program
|
|
1029
|
-
.storeOptionsAsProperties()
|
|
1030
|
-
.option('-d, --debug')
|
|
1031
|
-
.action((commandAndOptions) => {
|
|
1032
|
-
if (commandAndOptions.debug) {
|
|
1033
|
-
console.error(`Called ${commandAndOptions.name()}`);
|
|
1034
|
-
}
|
|
1035
|
-
});
|
|
1036
|
-
```
|
|
1037
|
-
|
|
1038
1018
|
### TypeScript
|
|
1039
1019
|
|
|
1040
1020
|
**extra-typings:** There is an optional project to infer extra type information from the option and argument definitions.
|
|
@@ -1075,7 +1055,7 @@ You can enable `--harmony` option in two ways:
|
|
|
1075
1055
|
|
|
1076
1056
|
An executable subcommand is launched as a separate child process.
|
|
1077
1057
|
|
|
1078
|
-
If you are using the node inspector for [debugging](https://nodejs.org/en/
|
|
1058
|
+
If you are using the node inspector for [debugging](https://nodejs.org/en/learn/getting-started/debugging) executable subcommands using `node --inspect` et al.,
|
|
1079
1059
|
the inspector port is incremented by 1 for the spawned subcommand.
|
|
1080
1060
|
|
|
1081
1061
|
If you are using VSCode to debug executable subcommands you need to set the `"autoAttachChildProcesses": true` flag in your `launch.json` configuration.
|
|
@@ -1164,9 +1144,3 @@ The current version of Commander is fully supported on Long Term Support version
|
|
|
1164
1144
|
Older major versions of Commander receive security updates for 12 months. For more see: [Release Policy](./docs/release-policy.md).
|
|
1165
1145
|
|
|
1166
1146
|
The main forum for free and community support is the project [Issues](https://github.com/tj/commander.js/issues) on GitHub.
|
|
1167
|
-
|
|
1168
|
-
### Commander for enterprise
|
|
1169
|
-
|
|
1170
|
-
Available as part of the Tidelift Subscription
|
|
1171
|
-
|
|
1172
|
-
The maintainers of Commander and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-commander?utm_source=npm-commander&utm_medium=referral&utm_campaign=enterprise&utm_term=repo)
|
|
@@ -18,4 +18,4 @@ export const createArgument = (name, description) =>
|
|
|
18
18
|
|
|
19
19
|
export { Command, Option, Argument, Help };
|
|
20
20
|
export { CommanderError, InvalidArgumentError };
|
|
21
|
-
export { InvalidArgumentError as InvalidOptionArgumentError };
|
|
21
|
+
export { InvalidArgumentError as InvalidOptionArgumentError };
|