@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -5
- package/dist/src/cli/commands/catalogue.d.ts +1 -2
- package/dist/src/cli/commands/catalogue.js +23 -15
- package/dist/src/cli/commands/portal.d.ts +0 -1
- package/dist/src/cli/commands/portal.js +13 -12
- package/dist/src/cli/index.d.ts +0 -1
- package/dist/src/cli/index.js +0 -1
- package/dist/src/cli/io.d.ts +2 -2
- package/dist/src/cli/io.js +6 -3
- package/dist/src/cli/program.d.ts +0 -1
- package/dist/src/cli/program.js +17 -11
- package/dist/src/cli/run.d.ts +27 -1
- package/dist/src/cli/run.js +90 -2
- package/dist/src/cli/shared.d.ts +18 -15
- package/dist/src/cli/shared.js +45 -54
- package/dist/src/client/client.d.ts +30 -9
- package/dist/src/client/client.js +154 -37
- package/dist/src/client/engine.d.ts +74 -13
- package/dist/src/client/engine.js +341 -71
- package/dist/src/client/errors.d.ts +44 -2
- package/dist/src/client/errors.js +94 -6
- package/dist/src/client/http.d.ts +23 -1
- package/dist/src/client/http.js +54 -33
- package/dist/src/client/index.d.ts +7 -6
- package/dist/src/client/index.js +5 -5
- package/dist/src/client/portals-list.d.ts +0 -1
- package/dist/src/client/portals-list.js +0 -1
- package/dist/src/client/portals.d.ts +35 -2
- package/dist/src/client/portals.js +86 -6
- package/dist/src/client/query.d.ts +0 -1
- package/dist/src/client/query.js +0 -1
- package/dist/src/client/types.d.ts +0 -1
- package/dist/src/client/types.js +0 -1
- package/dist/src/client/validate.d.ts +67 -0
- package/dist/src/client/validate.js +165 -0
- package/dist/src/index.d.ts +0 -1
- package/dist/src/index.js +0 -1
- package/package.json +4 -3
- package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
- package/dist/src/cli/commands/catalogue.js.map +0 -1
- package/dist/src/cli/commands/portal.d.ts.map +0 -1
- package/dist/src/cli/commands/portal.js.map +0 -1
- package/dist/src/cli/index.d.ts.map +0 -1
- package/dist/src/cli/index.js.map +0 -1
- package/dist/src/cli/io.d.ts.map +0 -1
- package/dist/src/cli/io.js.map +0 -1
- package/dist/src/cli/program.d.ts.map +0 -1
- package/dist/src/cli/program.js.map +0 -1
- package/dist/src/cli/run.d.ts.map +0 -1
- package/dist/src/cli/run.js.map +0 -1
- package/dist/src/cli/shared.d.ts.map +0 -1
- package/dist/src/cli/shared.js.map +0 -1
- package/dist/src/client/client.d.ts.map +0 -1
- package/dist/src/client/client.js.map +0 -1
- package/dist/src/client/engine.d.ts.map +0 -1
- package/dist/src/client/engine.js.map +0 -1
- package/dist/src/client/errors.d.ts.map +0 -1
- package/dist/src/client/errors.js.map +0 -1
- package/dist/src/client/http.d.ts.map +0 -1
- package/dist/src/client/http.js.map +0 -1
- package/dist/src/client/index.d.ts.map +0 -1
- package/dist/src/client/index.js.map +0 -1
- package/dist/src/client/portals-list.d.ts.map +0 -1
- package/dist/src/client/portals-list.js.map +0 -1
- package/dist/src/client/portals.d.ts.map +0 -1
- package/dist/src/client/portals.js.map +0 -1
- package/dist/src/client/query.d.ts.map +0 -1
- package/dist/src/client/query.js.map +0 -1
- package/dist/src/client/types.d.ts.map +0 -1
- package/dist/src/client/types.js.map +0 -1
- package/dist/src/index.d.ts.map +0 -1
- package/dist/src/index.js.map +0 -1
package/README.md
CHANGED
|
@@ -32,7 +32,7 @@ search, inspect, filter and pipe straight into [`jq`](https://jqlang.github.io/j
|
|
|
32
32
|
npm i -g @maschinenlesbar.org/ckan-cli
|
|
33
33
|
```
|
|
34
34
|
|
|
35
|
-
This installs the **`ckan`** command. Requires **Node.js
|
|
35
|
+
This installs the **`ckan`** command. Requires **Node.js 22.12+**.
|
|
36
36
|
|
|
37
37
|
## Quickstart
|
|
38
38
|
|
|
@@ -79,10 +79,16 @@ sub-path keeps it (`https://open.canada.ca/data`). `ckan status` is a good first
|
|
|
79
79
|
call on an unfamiliar portal: it shows the CKAN version and the installed
|
|
80
80
|
extensions.
|
|
81
81
|
|
|
82
|
+
A portal behind a login takes its user name and password in the URL
|
|
83
|
+
(`https://user:password@host`). They go out as HTTP Basic auth to that site only:
|
|
84
|
+
a redirect to another host, port or scheme drops them, and no message, help text
|
|
85
|
+
or error shows them.
|
|
86
|
+
|
|
82
87
|
### Known portals
|
|
83
88
|
|
|
84
89
|
`ckan portals` lists the CKAN portals in Germany this version knows, with the
|
|
85
90
|
result of their last check; `ckan portals --check` checks them all live now.
|
|
91
|
+
Neither uses `CKAN_BASE_URL`, so both work even when the variable is broken.
|
|
86
92
|
The list is built into the CLI, so it needs no outside service:
|
|
87
93
|
|
|
88
94
|
| `--portal` | Portal | Datasets |
|
|
@@ -152,7 +158,9 @@ action <name> [--param key=value …] call any CKAN action (generic)
|
|
|
152
158
|
> **Blank values are refused.** `ckan search ""`, `--fq ""`, `tags --query ""` or
|
|
153
159
|
> `action … --param q=` exit 1 before any request, instead of silently running an unfiltered search.
|
|
154
160
|
> So does a forgotten value (`--fq --rows 5`): a value of `--fq`, `--facet`,
|
|
155
|
-
> `--sort` or `tags --query` cannot start with `--`.
|
|
161
|
+
> `--sort` or `tags --query` cannot start with `--`. And so does a single-value
|
|
162
|
+
> option given twice (`--rows 5 --rows 50`, a second `--base-url`): only `--fq`,
|
|
163
|
+
> `--facet` and `--param` repeat.
|
|
156
164
|
|
|
157
165
|
### Paging and list flags
|
|
158
166
|
|
|
@@ -163,7 +171,9 @@ action <name> [--param key=value …] call any CKAN action (generic)
|
|
|
163
171
|
|
|
164
172
|
> **`--all-fields` lists are complete.** CKAN caps them at 25 entries
|
|
165
173
|
> (`ckan.group_and_organization_list_all_fields_max`) and drops the rest
|
|
166
|
-
> silently, even for a larger `--limit`; `ckan` pages through them for you
|
|
174
|
+
> silently, even for a larger `--limit`; `ckan` pages through them for you,
|
|
175
|
+
> 0.1 s apart. A server whose pages never end is stopped after 400 pages
|
|
176
|
+
> (10,000 entries, far more than any portal lists) with an error (exit 1).
|
|
167
177
|
| `tags` | `--query <substring>` |
|
|
168
178
|
|
|
169
179
|
### `action`
|
|
@@ -206,6 +216,9 @@ Found by running the same commands against many portals:
|
|
|
206
216
|
- **The default sort differs** (ZBW sorts by `metadata_created desc`).
|
|
207
217
|
- **Field types vary**: licence ids are URIs on GovData, and some portals send
|
|
208
218
|
booleans as the strings `"True"`/`"False"`.
|
|
219
|
+
- **A purely negative query matches nothing**: `ckan search -- -elbe` answers
|
|
220
|
+
`count: 0` on Hamburg rather than "everything but elbe". Negate a filter instead
|
|
221
|
+
(`--fq -organization:allris`), next to a positive query or none.
|
|
209
222
|
- **Solr syntax errors** come back as HTTP 409; `ckan` prints Solr's own reason,
|
|
210
223
|
e.g. `Search Error: Cannot parse 'title:(': Encountered "<EOF>" …`.
|
|
211
224
|
- **A site that is not (or no longer) a CKAN** answers with a 404, an HTML page or
|
|
@@ -235,7 +248,10 @@ ckan package bezirk-wandsbek-drucksache-22-3573-2 | jq '.resources[] | {format,
|
|
|
235
248
|
| --- | --- |
|
|
236
249
|
| `0` | success (also `--help`, `help [command]`, `--version` and a bare `ckan`) |
|
|
237
250
|
| `4` | not found (HTTP `404`): an unknown id on `package`/`resource`/`organization`/`group`, or, on any command, no CKAN at this URL (a wrong `--base-url`, a portal that left CKAN) |
|
|
238
|
-
| `1` | any other error — bad usage, validation error (`409`), unknown action (`400`), CKAN `success:false`, network failure |
|
|
251
|
+
| `1` | any other error — bad usage (an unknown command, also after `help`), validation error (`409`), unknown action (`400`), CKAN `success:false`, network failure |
|
|
252
|
+
|
|
253
|
+
A reader that stops early (`| head`) ends `ckan` quietly with exit 0; a failed
|
|
254
|
+
run keeps its exit code even when nothing reads stderr.
|
|
239
255
|
|
|
240
256
|
CKAN's error text is shown on stderr, e.g.
|
|
241
257
|
`Validation Error: rows: Invalid integer` or
|
|
@@ -258,7 +274,7 @@ CKAN's error text is shown on stderr, e.g.
|
|
|
258
274
|
| `--portal <id>` | a known portal by id (`ckan portals`); cannot be combined with `--base-url` |
|
|
259
275
|
| `--timeout <ms>` | Per-request timeout (default `30000`; at most `2147483647`) |
|
|
260
276
|
| `--user-agent <ua>` | `User-Agent` header value |
|
|
261
|
-
| `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry waits the server's `Retry-After`
|
|
277
|
+
| `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried |
|
|
262
278
|
| `--max-response-bytes <n>` | Cap response body size in bytes (`0` = unlimited; default 100 MiB) |
|
|
263
279
|
|
|
264
280
|
## Claude Code skills
|
|
@@ -1,26 +1,35 @@
|
|
|
1
1
|
// Commands over the catalogue: datasets, their resources, publishers, themes
|
|
2
2
|
// and tags.
|
|
3
|
-
import {
|
|
3
|
+
import { InvalidArgumentError } from "commander";
|
|
4
|
+
import { facetLimitProblem } from "../../client/validate.js";
|
|
5
|
+
import { action, collectNonEmpty, once, parseBoundedInt, parseIntArg, parseNonEmpty, parseOptionValue, renderJson, } from "../shared.js";
|
|
4
6
|
/**
|
|
5
7
|
* commander value-parser for a list --limit: 1 or more. CKAN reads `limit=0` as
|
|
6
8
|
* "no limit" and would send the whole list; leave --limit out for that.
|
|
7
9
|
*/
|
|
8
10
|
const parseLimit = parseBoundedInt(1, Number.MAX_SAFE_INTEGER);
|
|
9
|
-
/**
|
|
11
|
+
/**
|
|
12
|
+
* commander value-parser for --facet-limit: a count, or -1 for every value (the
|
|
13
|
+
* library's facetLimitProblem rule).
|
|
14
|
+
*/
|
|
10
15
|
function parseFacetLimit(value) {
|
|
11
|
-
|
|
16
|
+
const n = /^-\d+$/.test(value) ? Number(value) : parseIntArg(value);
|
|
17
|
+
const reason = facetLimitProblem(n);
|
|
18
|
+
if (reason !== undefined)
|
|
19
|
+
throw new InvalidArgumentError(reason);
|
|
20
|
+
return n;
|
|
12
21
|
}
|
|
13
22
|
export function registerCatalogueCommands(program, deps) {
|
|
14
23
|
program
|
|
15
24
|
.command("search")
|
|
16
25
|
.description("Search datasets (Solr query syntax)")
|
|
17
26
|
.argument("[query]", "full-text query, e.g. elbe or title:haushalt", parseNonEmpty)
|
|
18
|
-
.option("--rows <n>", "max results (servers cap this, usually at 1000)", parseIntArg)
|
|
19
|
-
.option("--start <n>", "offset for paging", parseIntArg)
|
|
20
|
-
.option("--sort <expr>", 'e.g. "metadata_modified desc"', parseOptionValue)
|
|
27
|
+
.option("--rows <n>", "max results (servers cap this, usually at 1000)", once(parseIntArg))
|
|
28
|
+
.option("--start <n>", "offset for paging", once(parseIntArg))
|
|
29
|
+
.option("--sort <expr>", 'e.g. "metadata_modified desc"', once(parseOptionValue))
|
|
21
30
|
.option("--fq <filter>", "filter query, e.g. organization:allris (repeatable; all must match)", collectNonEmpty)
|
|
22
31
|
.option("--facet <field>", "count values of a field, e.g. res_format (repeatable)", collectNonEmpty)
|
|
23
|
-
.option("--facet-limit <n>", "max values per facet (default 50; -1 = all)", parseFacetLimit)
|
|
32
|
+
.option("--facet-limit <n>", "max values per facet (default 50; -1 = all)", once(parseFacetLimit))
|
|
24
33
|
.action(action(deps, async ({ client, global, opts }, [query]) => {
|
|
25
34
|
renderJson(deps, global, await client.packageSearch({
|
|
26
35
|
q: query,
|
|
@@ -63,8 +72,8 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
63
72
|
program
|
|
64
73
|
.command("packages")
|
|
65
74
|
.description("List dataset names")
|
|
66
|
-
.option("--limit <n>", "max names (1 or more; omit for all)", parseLimit)
|
|
67
|
-
.option("--offset <n>", "offset for paging", parseIntArg)
|
|
75
|
+
.option("--limit <n>", "max names (1 or more; omit for all)", once(parseLimit))
|
|
76
|
+
.option("--offset <n>", "offset for paging", once(parseIntArg))
|
|
68
77
|
.action(action(deps, async ({ client, global, opts }) => {
|
|
69
78
|
renderJson(deps, global, await client.packageList({
|
|
70
79
|
limit: opts["limit"],
|
|
@@ -75,8 +84,8 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
75
84
|
.command("organizations")
|
|
76
85
|
.description("List organizations (data publishers)")
|
|
77
86
|
.option("--all-fields", "return full objects instead of names")
|
|
78
|
-
.option("--limit <n>", "max entries (1 or more; omit for all)", parseLimit)
|
|
79
|
-
.option("--offset <n>", "offset for paging", parseIntArg)
|
|
87
|
+
.option("--limit <n>", "max entries (1 or more; omit for all)", once(parseLimit))
|
|
88
|
+
.option("--offset <n>", "offset for paging", once(parseIntArg))
|
|
80
89
|
.action(action(deps, async ({ client, global, opts }) => {
|
|
81
90
|
renderJson(deps, global, await client.organizationList({
|
|
82
91
|
all_fields: opts["allFields"],
|
|
@@ -88,8 +97,8 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
88
97
|
.command("groups")
|
|
89
98
|
.description("List groups (themes/categories)")
|
|
90
99
|
.option("--all-fields", "return full objects instead of names")
|
|
91
|
-
.option("--limit <n>", "max entries (1 or more; omit for all)", parseLimit)
|
|
92
|
-
.option("--offset <n>", "offset for paging", parseIntArg)
|
|
100
|
+
.option("--limit <n>", "max entries (1 or more; omit for all)", once(parseLimit))
|
|
101
|
+
.option("--offset <n>", "offset for paging", once(parseIntArg))
|
|
93
102
|
.action(action(deps, async ({ client, global, opts }) => {
|
|
94
103
|
renderJson(deps, global, await client.groupList({
|
|
95
104
|
all_fields: opts["allFields"],
|
|
@@ -100,9 +109,8 @@ export function registerCatalogueCommands(program, deps) {
|
|
|
100
109
|
program
|
|
101
110
|
.command("tags")
|
|
102
111
|
.description("List tags")
|
|
103
|
-
.option("--query <substring>", "only tags containing this substring", parseOptionValue)
|
|
112
|
+
.option("--query <substring>", "only tags containing this substring", once(parseOptionValue))
|
|
104
113
|
.action(action(deps, async ({ client, global, opts }) => {
|
|
105
114
|
renderJson(deps, global, await client.tagList({ query: opts["query"] }));
|
|
106
115
|
}));
|
|
107
116
|
}
|
|
108
|
-
//# sourceMappingURL=catalogue.js.map
|
|
@@ -1,28 +1,29 @@
|
|
|
1
1
|
// Commands about the portal itself rather than its datasets.
|
|
2
2
|
import { InvalidArgumentError } from "commander";
|
|
3
3
|
import { action, renderJson, toEngineOptions } from "../shared.js";
|
|
4
|
-
import {
|
|
4
|
+
import { checkPortals } from "../../client/portals.js";
|
|
5
5
|
import { PORTALS } from "../../client/portals-list.js";
|
|
6
|
-
|
|
7
|
-
const CHECK_CONCURRENCY = 6;
|
|
6
|
+
import { isBlank } from "../../client/validate.js";
|
|
8
7
|
/** commander accumulator for repeatable `key=value` pairs into a record. */
|
|
9
8
|
function collectKeyValue(value, previous = {}) {
|
|
10
9
|
const eq = value.indexOf("=");
|
|
11
10
|
// Throw commander's InvalidArgumentError (not a CkanError) so the failure is
|
|
12
11
|
// formatted as a usage error with `error:` prefix and help, consistent with
|
|
13
12
|
// other parse-time flag validation (e.g. `--rows abc`).
|
|
14
|
-
|
|
13
|
+
const key = value.slice(0, Math.max(eq, 0));
|
|
14
|
+
// A blank key (`=x`, ` =x`) names no parameter: the library refuses it too.
|
|
15
|
+
if (eq < 0 || isBlank(key)) {
|
|
15
16
|
throw new InvalidArgumentError(`Invalid --param "${value}". Expected key=value.`);
|
|
16
17
|
}
|
|
17
|
-
const key = value.slice(0, eq);
|
|
18
18
|
// Reject a duplicated key rather than silently overwriting the earlier value.
|
|
19
19
|
if (Object.prototype.hasOwnProperty.call(previous, key)) {
|
|
20
20
|
throw new InvalidArgumentError(`Duplicate --param key "${key}".`);
|
|
21
21
|
}
|
|
22
|
-
// A blank value
|
|
23
|
-
// `--param q=` would
|
|
22
|
+
// A blank value is refused by the library as well (CKAN reads an empty
|
|
23
|
+
// parameter as unset, so `--param q=` would run the action unfiltered); here it
|
|
24
|
+
// is a usage error that names the flag.
|
|
24
25
|
const paramValue = value.slice(eq + 1);
|
|
25
|
-
if (paramValue
|
|
26
|
+
if (isBlank(paramValue)) {
|
|
26
27
|
throw new InvalidArgumentError(`Invalid --param "${value}". The value must not be blank.`);
|
|
27
28
|
}
|
|
28
29
|
// A computed key in a literal is an own data property, even for `__proto__`.
|
|
@@ -59,10 +60,10 @@ export function registerPortalCommands(program, deps) {
|
|
|
59
60
|
renderJson(deps, global, PORTALS);
|
|
60
61
|
return;
|
|
61
62
|
}
|
|
62
|
-
const
|
|
63
|
-
|
|
64
|
-
|
|
63
|
+
const checked = await checkPortals(PORTALS, {
|
|
64
|
+
engineOptions: toEngineOptions(global),
|
|
65
|
+
createClient: deps.createClient,
|
|
66
|
+
});
|
|
65
67
|
renderJson(deps, global, checked);
|
|
66
68
|
});
|
|
67
69
|
}
|
|
68
|
-
//# sourceMappingURL=portal.js.map
|
package/dist/src/cli/index.d.ts
CHANGED
package/dist/src/cli/index.js
CHANGED
package/dist/src/cli/io.d.ts
CHANGED
|
@@ -24,9 +24,9 @@ export interface OutputStreams {
|
|
|
24
24
|
* pager — closes the pipe while the CLI is still writing, and the next write fails
|
|
25
25
|
* with EPIPE. That is ordinary use, so the process exits 0 at once, quietly. Any
|
|
26
26
|
* other stdout error prints one `Output error: <message>` line to stderr and exits
|
|
27
|
-
* 1
|
|
27
|
+
* 1. On stderr an EPIPE is ignored, so a failed run keeps its exit code; any other
|
|
28
|
+
* stderr error exits 1 silently (there is nowhere left to report it).
|
|
28
29
|
* The bin shim installs this once, before `run()`.
|
|
29
30
|
*/
|
|
30
31
|
export declare function handleOutputErrors(streams?: OutputStreams, exit?: (code: number) => void): void;
|
|
31
32
|
export declare const defaultIO: CliIO;
|
|
32
|
-
//# sourceMappingURL=io.d.ts.map
|
package/dist/src/cli/io.js
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
* pager — closes the pipe while the CLI is still writing, and the next write fails
|
|
10
10
|
* with EPIPE. That is ordinary use, so the process exits 0 at once, quietly. Any
|
|
11
11
|
* other stdout error prints one `Output error: <message>` line to stderr and exits
|
|
12
|
-
* 1
|
|
12
|
+
* 1. On stderr an EPIPE is ignored, so a failed run keeps its exit code; any other
|
|
13
|
+
* stderr error exits 1 silently (there is nowhere left to report it).
|
|
13
14
|
* The bin shim installs this once, before `run()`.
|
|
14
15
|
*/
|
|
15
16
|
export function handleOutputErrors(streams = process, exit = (code) => process.exit(code)) {
|
|
@@ -19,12 +20,14 @@ export function handleOutputErrors(streams = process, exit = (code) => process.e
|
|
|
19
20
|
process.stderr.write(`Output error: ${err.message}\n`);
|
|
20
21
|
exit(1);
|
|
21
22
|
});
|
|
23
|
+
// stderr's reader going away doesn't make a failed run a success: ignore EPIPE there
|
|
24
|
+
// and let the run's own exit code stand (`2>&1 | true` used to turn a usage error into 0).
|
|
22
25
|
streams.stderr.on("error", (err) => {
|
|
23
|
-
|
|
26
|
+
if (err.code !== "EPIPE")
|
|
27
|
+
exit(1);
|
|
24
28
|
});
|
|
25
29
|
}
|
|
26
30
|
export const defaultIO = {
|
|
27
31
|
out: (text) => process.stdout.write(text + "\n"),
|
|
28
32
|
err: (text) => process.stderr.write(text + "\n"),
|
|
29
33
|
};
|
|
30
|
-
//# sourceMappingURL=io.js.map
|
package/dist/src/cli/program.js
CHANGED
|
@@ -6,10 +6,10 @@ import { fileURLToPath } from "node:url";
|
|
|
6
6
|
import { Command, InvalidArgumentError, Option } from "commander";
|
|
7
7
|
import { defaultIO } from "./io.js";
|
|
8
8
|
import { CkanClient } from "../client/client.js";
|
|
9
|
-
import { DEFAULT_BASE_URL, MAX_RETRIES } from "../client/engine.js";
|
|
9
|
+
import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
|
|
10
10
|
import { MAX_TIMEOUT_MS } from "../client/http.js";
|
|
11
11
|
import { redactUrl } from "../client/errors.js";
|
|
12
|
-
import { parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
|
|
12
|
+
import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
|
|
13
13
|
import { registerCatalogueCommands } from "./commands/catalogue.js";
|
|
14
14
|
import { registerPortalCommands } from "./commands/portal.js";
|
|
15
15
|
/**
|
|
@@ -43,17 +43,17 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
43
43
|
.description("CLI for any CKAN open-data portal (Action API v3)")
|
|
44
44
|
.version(VERSION)
|
|
45
45
|
.addOption(new Option("--base-url <url>", "CKAN site URL (env CKAN_BASE_URL)")
|
|
46
|
-
.argParser(parseBaseUrl)
|
|
46
|
+
.argParser(once(parseBaseUrl))
|
|
47
47
|
// The help shows the default without userinfo: a password in
|
|
48
48
|
// CKAN_BASE_URL must not end up in `--help` output or CI logs.
|
|
49
49
|
.default(baseUrlDefault, JSON.stringify(redactUrl(baseUrlDefault))))
|
|
50
50
|
.addOption(new Option("--portal <id>", "a known portal by id (see `ckan portals`)")
|
|
51
|
-
.argParser(parsePortal)
|
|
51
|
+
.argParser(once(parsePortal))
|
|
52
52
|
.conflicts("baseUrl"))
|
|
53
|
-
.option("--timeout <ms>", "per-request timeout in milliseconds", parseBoundedInt(0, MAX_TIMEOUT_MS))
|
|
54
|
-
.option("--user-agent <ua>", "User-Agent header value", parseHeaderValue)
|
|
55
|
-
.option("--max-retries <n>",
|
|
56
|
-
.option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", parseIntArg)
|
|
53
|
+
.option("--timeout <ms>", "per-request timeout in milliseconds", once(parseBoundedInt(0, MAX_TIMEOUT_MS)))
|
|
54
|
+
.option("--user-agent <ua>", "User-Agent header value", once(parseHeaderValue))
|
|
55
|
+
.option("--max-retries <n>", `retries for transient 429/503 responses (0..${MAX_RETRIES}; each backs off linearly, or waits a longer Retry-After up to ${MAX_RETRY_AFTER_MS / 1000} s)`, once(parseBoundedInt(0, MAX_RETRIES)))
|
|
56
|
+
.option("--max-response-bytes <n>", "cap response body size in bytes (0 = unlimited; default 100 MiB)", once(parseIntArg))
|
|
57
57
|
.option("--compact", "print JSON on a single line instead of pretty-printed")
|
|
58
58
|
.showHelpAfterError();
|
|
59
59
|
// A bare invocation (no subcommand) should print help to stdout and exit 0,
|
|
@@ -73,8 +73,15 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
73
73
|
program.help();
|
|
74
74
|
});
|
|
75
75
|
// commander runs value parsers on flags but not on defaults, so a base URL
|
|
76
|
-
// taken from CKAN_BASE_URL is checked here, before
|
|
77
|
-
|
|
76
|
+
// taken from CKAN_BASE_URL is checked here, before a command that uses it runs.
|
|
77
|
+
// The commands that never use it skip the check: help (the root action prints
|
|
78
|
+
// help or names an unknown command, `help` prints help) and `portals`, which
|
|
79
|
+
// prints the built-in list or checks each portal at its own URL — a user whose
|
|
80
|
+
// variable is broken is exactly the one looking for a working portal id.
|
|
81
|
+
const offline = new Set(["help", "portals"]);
|
|
82
|
+
program.hook("preAction", (_program, actionCommand) => {
|
|
83
|
+
if (actionCommand === program || offline.has(actionCommand.name()))
|
|
84
|
+
return;
|
|
78
85
|
if (program.getOptionValueSource("baseUrl") !== "default" || program.opts()["portal"] !== undefined)
|
|
79
86
|
return;
|
|
80
87
|
try {
|
|
@@ -93,4 +100,3 @@ export function buildProgram(deps = defaultDeps) {
|
|
|
93
100
|
program.allowExcessArguments(true);
|
|
94
101
|
return program;
|
|
95
102
|
}
|
|
96
|
-
//# sourceMappingURL=program.js.map
|
package/dist/src/cli/run.d.ts
CHANGED
|
@@ -1,3 +1,29 @@
|
|
|
1
|
+
import { type Command } from "commander";
|
|
1
2
|
import type { CliDeps } from "./io.js";
|
|
3
|
+
/**
|
|
4
|
+
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
5
|
+
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
6
|
+
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
7
|
+
* `withRedactedOutput`.
|
|
8
|
+
*/
|
|
9
|
+
export declare function redactUserinfo(text: string): string;
|
|
10
|
+
/**
|
|
11
|
+
* `deps` with an `io` that redacts the credentials of every argument and of
|
|
12
|
+
* CKAN_BASE_URL from everything it prints. Commander echoes rejected values in its
|
|
13
|
+
* errors (`--base-url`, `--portal`, a URL typed where the command goes), the
|
|
14
|
+
* client's own messages name the values they refuse, and help shows the base URL's
|
|
15
|
+
* default: whatever path a credential takes to stdout or stderr, the exact userinfo
|
|
16
|
+
* (as `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is
|
|
17
|
+
* replaced by `***`. A pattern alone can't delimit a password with spaces, quotes,
|
|
18
|
+
* `#`, `?` or `/`; the exact strings can. Without credentials the output passes
|
|
19
|
+
* through unchanged.
|
|
20
|
+
*/
|
|
21
|
+
export declare function withRedactedOutput(deps: CliDeps, argv: readonly string[]): CliDeps;
|
|
22
|
+
/**
|
|
23
|
+
* The index of a `help` command whose topic names no command (`ckan help nonexistent`),
|
|
24
|
+
* or undefined. Commander answers that with the root help on stderr and exit 1, but
|
|
25
|
+
* no line saying what was wrong. Global options before `help` are skipped with their
|
|
26
|
+
* values; scanning stops at `--`.
|
|
27
|
+
*/
|
|
28
|
+
export declare function unknownHelpTopic(program: Command, argv: readonly string[]): number | undefined;
|
|
2
29
|
export declare function run(argv: string[], deps?: CliDeps): Promise<number>;
|
|
3
|
-
//# sourceMappingURL=run.d.ts.map
|
package/dist/src/cli/run.js
CHANGED
|
@@ -3,7 +3,9 @@
|
|
|
3
3
|
// captured output and exit code without spawning a subprocess.
|
|
4
4
|
import { CommanderError } from "commander";
|
|
5
5
|
import { buildProgram, defaultDeps } from "./program.js";
|
|
6
|
-
import { CkanApiError, CkanError } from "../client/errors.js";
|
|
6
|
+
import { CkanApiError, CkanError, CkanValidationError, credentialsIn, redactCredentials, } from "../client/errors.js";
|
|
7
|
+
/** The exit code of a usage error: commander's own for a rejected option value. */
|
|
8
|
+
const USAGE_EXIT = 1;
|
|
7
9
|
/**
|
|
8
10
|
* Apply exitOverride + output redirection to every command in the tree.
|
|
9
11
|
* commander does not propagate these to subcommands, so a parse error on a
|
|
@@ -18,9 +20,90 @@ function configureTree(command, deps) {
|
|
|
18
20
|
for (const child of command.commands)
|
|
19
21
|
configureTree(child, deps);
|
|
20
22
|
}
|
|
23
|
+
/**
|
|
24
|
+
* Replace the userinfo of every URL in `text` with `***`, the form `redactUrl` gives
|
|
25
|
+
* (`https://user:secret@host` becomes `https://***@host`). Text-based, so it also
|
|
26
|
+
* covers a URL that does not parse; a fallback behind the exact strings of
|
|
27
|
+
* `withRedactedOutput`.
|
|
28
|
+
*/
|
|
29
|
+
export function redactUserinfo(text) {
|
|
30
|
+
return text.replace(/\b([a-z][a-z0-9+.-]*:\/\/)[^\s/?#']*@/gi, "$1***@");
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* `deps` with an `io` that redacts the credentials of every argument and of
|
|
34
|
+
* CKAN_BASE_URL from everything it prints. Commander echoes rejected values in its
|
|
35
|
+
* errors (`--base-url`, `--portal`, a URL typed where the command goes), the
|
|
36
|
+
* client's own messages name the values they refuse, and help shows the base URL's
|
|
37
|
+
* default: whatever path a credential takes to stdout or stderr, the exact userinfo
|
|
38
|
+
* (as `credentialsIn` finds it, plus its JSON-escaped and percent-encoded forms) is
|
|
39
|
+
* replaced by `***`. A pattern alone can't delimit a password with spaces, quotes,
|
|
40
|
+
* `#`, `?` or `/`; the exact strings can. Without credentials the output passes
|
|
41
|
+
* through unchanged.
|
|
42
|
+
*/
|
|
43
|
+
export function withRedactedOutput(deps, argv) {
|
|
44
|
+
// An `--option=value` token is echoed as its value alone.
|
|
45
|
+
const values = argv.map((token) => (token.startsWith("-") && token.includes("=") ? token.slice(token.indexOf("=") + 1) : token));
|
|
46
|
+
const sources = [...argv, ...values, deps.env["CKAN_BASE_URL"] ?? ""];
|
|
47
|
+
const secrets = new Set();
|
|
48
|
+
const encoded = new Set();
|
|
49
|
+
for (const source of sources) {
|
|
50
|
+
for (const secret of credentialsIn(source)) {
|
|
51
|
+
secrets.add(secret);
|
|
52
|
+
secrets.add(JSON.stringify(secret).slice(1, -1));
|
|
53
|
+
// A URL sent as a parameter (`package <url>`) is echoed percent-encoded in a
|
|
54
|
+
// 404 message: `id=https%3A%2F%2Fuser%3Apw%40host`.
|
|
55
|
+
encoded.add(encodeURIComponent(`${secret}@`));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
if (secrets.size === 0)
|
|
59
|
+
return deps;
|
|
60
|
+
const list = [...secrets];
|
|
61
|
+
const redact = (text) => {
|
|
62
|
+
let out = redactUserinfo(redactCredentials(text, list));
|
|
63
|
+
for (const form of encoded)
|
|
64
|
+
out = out.split(form).join("***%40");
|
|
65
|
+
return out;
|
|
66
|
+
};
|
|
67
|
+
return { ...deps, io: { out: (text) => deps.io.out(redact(text)), err: (text) => deps.io.err(redact(text)) } };
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The index of a `help` command whose topic names no command (`ckan help nonexistent`),
|
|
71
|
+
* or undefined. Commander answers that with the root help on stderr and exit 1, but
|
|
72
|
+
* no line saying what was wrong. Global options before `help` are skipped with their
|
|
73
|
+
* values; scanning stops at `--`.
|
|
74
|
+
*/
|
|
75
|
+
export function unknownHelpTopic(program, argv) {
|
|
76
|
+
for (let i = 0; i < argv.length; i++) {
|
|
77
|
+
const token = argv[i];
|
|
78
|
+
if (token === "--")
|
|
79
|
+
return undefined;
|
|
80
|
+
if (token.startsWith("-")) {
|
|
81
|
+
if (token.includes("="))
|
|
82
|
+
continue;
|
|
83
|
+
const option = program.options.find((o) => o.long === token || o.short === token);
|
|
84
|
+
if (option?.required)
|
|
85
|
+
i++;
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
88
|
+
if (token !== "help")
|
|
89
|
+
return undefined;
|
|
90
|
+
const topic = argv[i + 1];
|
|
91
|
+
if (topic === undefined || topic.startsWith("-"))
|
|
92
|
+
return undefined;
|
|
93
|
+
const known = program.commands.some((c) => c.name() === topic || c.aliases().includes(topic));
|
|
94
|
+
return known ? undefined : i;
|
|
95
|
+
}
|
|
96
|
+
return undefined;
|
|
97
|
+
}
|
|
21
98
|
export async function run(argv, deps = defaultDeps) {
|
|
99
|
+
deps = withRedactedOutput(deps, argv);
|
|
22
100
|
const program = buildProgram(deps);
|
|
23
101
|
configureTree(program, deps);
|
|
102
|
+
// `help <unknown>` is answered like `<unknown>`: "error: unknown command '<unknown>'",
|
|
103
|
+
// the help after it, exit 1 — not the help alone with a failure code.
|
|
104
|
+
const helpAt = unknownHelpTopic(program, argv);
|
|
105
|
+
if (helpAt !== undefined)
|
|
106
|
+
argv = argv.filter((_, i) => i !== helpAt);
|
|
24
107
|
try {
|
|
25
108
|
await program.parseAsync(argv, { from: "user" });
|
|
26
109
|
return 0;
|
|
@@ -30,6 +113,12 @@ export async function run(argv, deps = defaultDeps) {
|
|
|
30
113
|
// Help/version requests exit 0; genuine parse errors carry their own code.
|
|
31
114
|
return err.exitCode;
|
|
32
115
|
}
|
|
116
|
+
if (err instanceof CkanValidationError) {
|
|
117
|
+
// An input the library refused before any request (a rule the commander
|
|
118
|
+
// parsers do not see, e.g. one across parameters): a usage error.
|
|
119
|
+
deps.io.err(`Error: ${err.message}`);
|
|
120
|
+
return USAGE_EXIT;
|
|
121
|
+
}
|
|
33
122
|
if (err instanceof CkanApiError) {
|
|
34
123
|
deps.io.err(`Error: ${err.message}`);
|
|
35
124
|
// Map a few notable statuses to distinct exit codes for scripting.
|
|
@@ -45,4 +134,3 @@ export async function run(argv, deps = defaultDeps) {
|
|
|
45
134
|
return 1;
|
|
46
135
|
}
|
|
47
136
|
}
|
|
48
|
-
//# sourceMappingURL=run.js.map
|
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -3,17 +3,14 @@ import { type EngineOptions } from "../client/engine.js";
|
|
|
3
3
|
/** commander value-parser: a non-negative integer. */
|
|
4
4
|
export declare function parseIntArg(value: string): number;
|
|
5
5
|
/**
|
|
6
|
-
* commander value-parser: a value that is not blank
|
|
7
|
-
*
|
|
6
|
+
* commander value-parser: a value that is not blank (the library's blankProblem
|
|
7
|
+
* rule, reported as a usage error before the client is built).
|
|
8
8
|
*/
|
|
9
9
|
export declare function parseNonEmpty(value: string): string;
|
|
10
10
|
/**
|
|
11
|
-
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`)
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* U+00FF, which surfaced as "Unexpected error". Reject those here as a usage error,
|
|
15
|
-
* along with a blank value. Tab is allowed, as in HTTP. Checked by char code so the
|
|
16
|
-
* source stays free of control bytes.
|
|
11
|
+
* commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
|
|
12
|
+
* the library's headerValueProblem rule (not blank, no control characters but tab,
|
|
13
|
+
* nothing above U+00FF), reported as a usage error.
|
|
17
14
|
*/
|
|
18
15
|
export declare function parseHeaderValue(value: string): string;
|
|
19
16
|
/**
|
|
@@ -25,16 +22,23 @@ export declare function parseHeaderValue(value: string): string;
|
|
|
25
22
|
* (a negated Solr filter, `-organization:x`) stays allowed.
|
|
26
23
|
*/
|
|
27
24
|
export declare function parseOptionValue(value: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* Wrap a commander value-parser for a single-valued option so that a second
|
|
27
|
+
* occurrence is a usage error. commander otherwise keeps only the last value, so
|
|
28
|
+
* `--rows 5 --rows 50` (or a second `--base-url`) silently dropped the first. The
|
|
29
|
+
* wrapper keeps its own count, so it also works for an option with a default (which
|
|
30
|
+
* commander hands the parser as `previous`); build it per program (`buildProgram`).
|
|
31
|
+
*/
|
|
32
|
+
export declare function once<T>(parse: (value: string) => T): (value: string) => T;
|
|
28
33
|
/** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
|
|
29
34
|
export declare function collectNonEmpty(value: string, previous?: string[]): string[];
|
|
30
|
-
/** Build a commander value-parser for a non-negative integer within [min, max]. */
|
|
35
|
+
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
31
36
|
export declare function parseBoundedInt(min: number, max: number): (value: string) => number;
|
|
32
37
|
/**
|
|
33
|
-
* commander value-parser for --base-url
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
* but this surfaces the mistake up front and independently of the transport.
|
|
38
|
+
* commander value-parser for --base-url (and CKAN_BASE_URL, via the preAction hook
|
|
39
|
+
* in program.ts): the library's baseUrlProblem rule (not blank, no whitespace, an
|
|
40
|
+
* absolute http(s) URL, no query string or fragment), reported as a usage error
|
|
41
|
+
* before the client is built.
|
|
38
42
|
*/
|
|
39
43
|
export declare function parseBaseUrl(value: string): string;
|
|
40
44
|
/** commander value-parser for --portal: the URL of a portal in the built-in list. */
|
|
@@ -77,4 +81,3 @@ export interface ActionContext {
|
|
|
77
81
|
* the trailing options object and command instance to recover the positionals.
|
|
78
82
|
*/
|
|
79
83
|
export declare function action(deps: CliDeps, fn: (ctx: ActionContext, positionals: string[]) => Promise<void>): (...args: unknown[]) => Promise<void>;
|
|
80
|
-
//# sourceMappingURL=shared.d.ts.map
|