@maschinenlesbar.org/ckan-cli 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +55 -12
- package/dist/src/cli/commands/catalogue.js +12 -12
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -5
- package/dist/src/cli/io.js +65 -9
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +88 -11
- package/dist/src/cli/run.d.ts +53 -1
- package/dist/src/cli/run.js +239 -8
- package/dist/src/cli/shared.d.ts +17 -1
- package/dist/src/cli/shared.js +33 -3
- package/dist/src/client/client.d.ts +23 -3
- package/dist/src/client/client.js +124 -22
- package/dist/src/client/engine.d.ts +62 -9
- package/dist/src/client/engine.js +350 -47
- package/dist/src/client/errors.d.ts +81 -1
- package/dist/src/client/errors.js +171 -5
- package/dist/src/client/http.d.ts +23 -0
- package/dist/src/client/http.js +14 -1
- package/dist/src/client/index.d.ts +3 -3
- package/dist/src/client/index.js +3 -3
- package/dist/src/client/portals.d.ts +1 -1
- package/dist/src/client/portals.js +39 -4
- package/dist/src/client/validate.d.ts +13 -1
- package/dist/src/client/validate.js +49 -3
- package/package.json +6 -5
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type
|
|
1
|
+
import { type CliDeps } from "./io.js";
|
|
2
2
|
import { type EngineOptions } from "../client/engine.js";
|
|
3
3
|
/** commander value-parser: a non-negative integer. */
|
|
4
4
|
export declare function parseIntArg(value: string): number;
|
|
@@ -22,6 +22,14 @@ export declare function parseHeaderValue(value: string): string;
|
|
|
22
22
|
* (a negated Solr filter, `-organization:x`) stays allowed.
|
|
23
23
|
*/
|
|
24
24
|
export declare function parseOptionValue(value: string): string;
|
|
25
|
+
/**
|
|
26
|
+
* Wrap a commander value-parser for a single-valued option so that a second
|
|
27
|
+
* occurrence is a usage error. commander otherwise keeps only the last value, so
|
|
28
|
+
* `--rows 5 --rows 50` (or a second `--base-url`) silently dropped the first. The
|
|
29
|
+
* wrapper keeps its own count, so it also works for an option with a default (which
|
|
30
|
+
* commander hands the parser as `previous`); build it per program (`buildProgram`).
|
|
31
|
+
*/
|
|
32
|
+
export declare function once<T>(parse: (value: string) => T): (value: string) => T;
|
|
25
33
|
/** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
|
|
26
34
|
export declare function collectNonEmpty(value: string, previous?: string[]): string[];
|
|
27
35
|
/** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
|
|
@@ -58,6 +66,14 @@ export declare function toEngineOptions(global: GlobalOptions): EngineOptions;
|
|
|
58
66
|
export declare function escapeControlChars(json: string): string;
|
|
59
67
|
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
60
68
|
export declare function renderJson(deps: CliDeps, global: GlobalOptions, value: unknown): void;
|
|
69
|
+
/**
|
|
70
|
+
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
71
|
+
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
72
|
+
* (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
|
|
73
|
+
* Called once per run, after the options are parsed and before the first request;
|
|
74
|
+
* stdout and the exit code are untouched.
|
|
75
|
+
*/
|
|
76
|
+
export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
|
|
61
77
|
export interface ActionContext {
|
|
62
78
|
client: ReturnType<CliDeps["createClient"]>;
|
|
63
79
|
global: GlobalOptions;
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Shared helpers used across CLI command groups: option parsers, the global
|
|
2
2
|
// option resolver, and the JSON result renderer.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { logOf } from "./io.js";
|
|
5
|
+
import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "../client/engine.js";
|
|
6
|
+
import { CkanError, cutForMessage, redactUrl } from "../client/errors.js";
|
|
6
7
|
import { findPortal } from "../client/portals.js";
|
|
7
8
|
import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
|
|
8
9
|
import { PORTALS } from "../client/portals-list.js";
|
|
@@ -57,6 +58,22 @@ export function parseOptionValue(value) {
|
|
|
57
58
|
}
|
|
58
59
|
return parseNonEmpty(value);
|
|
59
60
|
}
|
|
61
|
+
/**
|
|
62
|
+
* Wrap a commander value-parser for a single-valued option so that a second
|
|
63
|
+
* occurrence is a usage error. commander otherwise keeps only the last value, so
|
|
64
|
+
* `--rows 5 --rows 50` (or a second `--base-url`) silently dropped the first. The
|
|
65
|
+
* wrapper keeps its own count, so it also works for an option with a default (which
|
|
66
|
+
* commander hands the parser as `previous`); build it per program (`buildProgram`).
|
|
67
|
+
*/
|
|
68
|
+
export function once(parse) {
|
|
69
|
+
let seen = false;
|
|
70
|
+
return (value) => {
|
|
71
|
+
if (seen)
|
|
72
|
+
throw new InvalidArgumentError("Given more than once; this option takes a single value.");
|
|
73
|
+
seen = true;
|
|
74
|
+
return parse(value);
|
|
75
|
+
};
|
|
76
|
+
}
|
|
60
77
|
/** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
|
|
61
78
|
export function collectNonEmpty(value, previous = []) {
|
|
62
79
|
return previous.concat([parseOptionValue(value)]);
|
|
@@ -88,7 +105,7 @@ export function parseBaseUrl(value) {
|
|
|
88
105
|
export function parsePortal(value) {
|
|
89
106
|
const portal = findPortal(value, PORTALS);
|
|
90
107
|
if (!portal) {
|
|
91
|
-
throw new InvalidArgumentError(`Unknown portal "${value}". \`ckan portals\` lists the known ones.`);
|
|
108
|
+
throw new InvalidArgumentError(`Unknown portal "${cutForMessage(redactUrl(value))}". \`ckan portals\` lists the known ones.`);
|
|
92
109
|
}
|
|
93
110
|
return portal.url;
|
|
94
111
|
}
|
|
@@ -153,6 +170,18 @@ export function renderJson(deps, global, value) {
|
|
|
153
170
|
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
154
171
|
deps.io.out(text);
|
|
155
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
175
|
+
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
176
|
+
* (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
|
|
177
|
+
* Called once per run, after the options are parsed and before the first request;
|
|
178
|
+
* stdout and the exit code are untouched.
|
|
179
|
+
*/
|
|
180
|
+
export function warnOnCleartext(deps, global) {
|
|
181
|
+
const problem = cleartextProblem(global.portal ?? global.baseUrl ?? DEFAULT_BASE_URL);
|
|
182
|
+
if (problem !== undefined)
|
|
183
|
+
logOf(deps).warn("http", problem);
|
|
184
|
+
}
|
|
156
185
|
/**
|
|
157
186
|
* Wrap an async command action with consistent global-option resolution and
|
|
158
187
|
* client construction. The callback receives a context (client + resolved global
|
|
@@ -166,6 +195,7 @@ export function action(deps, fn) {
|
|
|
166
195
|
const command = args[args.length - 1];
|
|
167
196
|
const positionals = args.slice(0, Math.max(0, args.length - 2));
|
|
168
197
|
const global = command.optsWithGlobals();
|
|
198
|
+
warnOnCleartext(deps, global);
|
|
169
199
|
const client = deps.createClient(toEngineOptions(global));
|
|
170
200
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
171
201
|
};
|
|
@@ -1,6 +1,19 @@
|
|
|
1
1
|
import { type EngineOptions } from "./engine.js";
|
|
2
2
|
import type { QueryParams } from "./query.js";
|
|
3
3
|
import type { Group, GroupListParams, JsonValue, License, ListParams, Organization, Package, PackageSearchParams, PackageSearchResult, Resource, Status, TagListParams } from "./types.js";
|
|
4
|
+
/**
|
|
5
|
+
* Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
|
|
6
|
+
* of 25). Real CKAN portals list a few hundred organizations or groups at most; a
|
|
7
|
+
* server whose pages keep returning new entries (a broken or hostile one) would
|
|
8
|
+
* otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
|
|
9
|
+
*/
|
|
10
|
+
export declare const MAX_ALL_FIELDS_PAGES = 400;
|
|
11
|
+
/**
|
|
12
|
+
* Pause between two `all_fields` pages, in milliseconds. A long list costs a few
|
|
13
|
+
* tenths of a second more; a server that never ends the list gets 10 requests a
|
|
14
|
+
* second at most instead of thousands.
|
|
15
|
+
*/
|
|
16
|
+
export declare const ALL_FIELDS_PAGE_DELAY_MS = 100;
|
|
4
17
|
/**
|
|
5
18
|
* The site root of a CKAN instance, from what a user is likely to paste: portals
|
|
6
19
|
* document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
|
|
@@ -10,9 +23,7 @@ import type { Group, GroupListParams, JsonValue, License, ListParams, Organizati
|
|
|
10
23
|
*/
|
|
11
24
|
export declare function siteRoot(baseUrl: string): string;
|
|
12
25
|
export declare class CkanClient {
|
|
13
|
-
private
|
|
14
|
-
/** The site this client talks to, for error messages. */
|
|
15
|
-
private readonly site;
|
|
26
|
+
#private;
|
|
16
27
|
constructor(options?: EngineOptions);
|
|
17
28
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
18
29
|
action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
|
|
@@ -68,6 +79,15 @@ export declare class CkanClient {
|
|
|
68
79
|
* some entries), so only an empty page ends the list. Entries are deduplicated
|
|
69
80
|
* (by `id`), and a page that adds nothing new ends the loop too, so a server
|
|
70
81
|
* that ignores `offset` cannot keep it going.
|
|
82
|
+
*
|
|
83
|
+
* Berlin counts hidden entries in its `limit`/`offset` window but leaves them
|
|
84
|
+
* out of the answer, so a window smaller than a full page (the last one before
|
|
85
|
+
* `limit` is reached) can come back empty although more entries follow. Such a
|
|
86
|
+
* miss is asked again at the same offset with a full page before it ends the list.
|
|
87
|
+
*
|
|
88
|
+
* Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
|
|
89
|
+
* `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
|
|
90
|
+
* keep returning new entries) the call fails with a CkanParseError.
|
|
71
91
|
*/
|
|
72
92
|
private groupOrOrgList;
|
|
73
93
|
/**
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
3
|
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
-
import {
|
|
5
|
-
import { assertValid, blankProblem, countProblem, facetLimitProblem } from "./validate.js";
|
|
4
|
+
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
+
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
7
|
/**
|
|
8
8
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
@@ -13,6 +13,20 @@ const ACTION = "/api/3/action";
|
|
|
13
13
|
const ACTION_NAME = /^[a-z0-9_]+$/;
|
|
14
14
|
/** CKAN's default cap on an `all_fields` organization/group list. */
|
|
15
15
|
const ALL_FIELDS_PAGE = 25;
|
|
16
|
+
/**
|
|
17
|
+
* Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
|
|
18
|
+
* of 25). Real CKAN portals list a few hundred organizations or groups at most; a
|
|
19
|
+
* server whose pages keep returning new entries (a broken or hostile one) would
|
|
20
|
+
* otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
|
|
21
|
+
*/
|
|
22
|
+
export const MAX_ALL_FIELDS_PAGES = 400;
|
|
23
|
+
/**
|
|
24
|
+
* Pause between two `all_fields` pages, in milliseconds. A long list costs a few
|
|
25
|
+
* tenths of a second more; a server that never ends the list gets 10 requests a
|
|
26
|
+
* second at most instead of thousands.
|
|
27
|
+
*/
|
|
28
|
+
export const ALL_FIELDS_PAGE_DELAY_MS = 100;
|
|
29
|
+
const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
16
30
|
/** A key for a list entry — its `id`, else its JSON — to drop duplicates across pages. */
|
|
17
31
|
function entryKey(entry) {
|
|
18
32
|
if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
|
|
@@ -29,6 +43,7 @@ function entryKey(entry) {
|
|
|
29
43
|
* filter such as `tagList({ query: "" })` would silently return everything.
|
|
30
44
|
*/
|
|
31
45
|
function prune(params) {
|
|
46
|
+
assertParams(params);
|
|
32
47
|
// A null-prototype object, so a `__proto__` key is kept as a parameter instead
|
|
33
48
|
// of setting the prototype (and being lost).
|
|
34
49
|
const out = Object.create(null);
|
|
@@ -36,10 +51,46 @@ function prune(params) {
|
|
|
36
51
|
if (v === undefined)
|
|
37
52
|
continue;
|
|
38
53
|
assertValid("parameter name", k, blankProblem);
|
|
39
|
-
|
|
54
|
+
// A value the query string can't carry (an object, NaN, a function) is refused,
|
|
55
|
+
// not sent as "[object Object]" or "NaN".
|
|
56
|
+
out[k] = assertValid(k, v, queryValueProblem);
|
|
40
57
|
}
|
|
41
58
|
return out;
|
|
42
59
|
}
|
|
60
|
+
/**
|
|
61
|
+
* A method's parameter object: a plain object (or undefined, for the defaults).
|
|
62
|
+
* `null`, a string or an array would otherwise fail as a raw TypeError.
|
|
63
|
+
*/
|
|
64
|
+
function assertParams(params) {
|
|
65
|
+
if (params === undefined)
|
|
66
|
+
return;
|
|
67
|
+
if (typeof params !== "object" || params === null || Array.isArray(params)) {
|
|
68
|
+
throw new CkanValidationError("Invalid params: Expected an object.");
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Throw CkanValidationError unless every own key of `params` is in `known`. CKAN
|
|
73
|
+
* ignores a parameter it doesn't know, so a misspelled `fqs` or a JSON `__proto__`
|
|
74
|
+
* key would run the search unfiltered over the whole catalogue. Any other CKAN
|
|
75
|
+
* parameter goes through `action(name, params)`, which sends every key.
|
|
76
|
+
*/
|
|
77
|
+
function assertKeys(method, action, params, known) {
|
|
78
|
+
for (const key of Object.keys(params)) {
|
|
79
|
+
if (!known.includes(key)) {
|
|
80
|
+
throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
|
|
81
|
+
`Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_limit"];
|
|
86
|
+
const PACKAGE_LIST_KEYS = ["limit", "offset"];
|
|
87
|
+
const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
|
|
88
|
+
const TAG_LIST_KEYS = ["query"];
|
|
89
|
+
/** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
|
|
90
|
+
function assertText(name, value) {
|
|
91
|
+
if (value !== undefined)
|
|
92
|
+
assertValid(name, value, textProblem);
|
|
93
|
+
}
|
|
43
94
|
/** A JSON object (not null, not an array). */
|
|
44
95
|
function isObject(value) {
|
|
45
96
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
@@ -85,28 +136,36 @@ export function siteRoot(baseUrl) {
|
|
|
85
136
|
return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
|
|
86
137
|
}
|
|
87
138
|
export class CkanClient {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
139
|
+
// Real private fields (not TypeScript's `private`): console.log, util.inspect and
|
|
140
|
+
// JSON.stringify of a client never show them, so a password in the base URL can't
|
|
141
|
+
// be logged by accident.
|
|
142
|
+
#engine;
|
|
143
|
+
/** The site this client talks to, for error messages (always through redactUrl). */
|
|
144
|
+
#site;
|
|
145
|
+
/** The pause between `all_fields` pages (EngineOptions.sleep, for tests). */
|
|
146
|
+
#sleep;
|
|
91
147
|
constructor(options = {}) {
|
|
92
|
-
this
|
|
93
|
-
|
|
148
|
+
this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
149
|
+
// The engine checks every option, `sleep` included, before the client uses it.
|
|
150
|
+
this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
|
|
151
|
+
this.#sleep = options.sleep ?? realSleep;
|
|
94
152
|
}
|
|
95
153
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
96
154
|
async action(name, params = {}) {
|
|
97
|
-
if (!ACTION_NAME.test(name)) {
|
|
98
|
-
throw new
|
|
155
|
+
if (typeof name !== "string" || !ACTION_NAME.test(name)) {
|
|
156
|
+
throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
|
|
99
157
|
}
|
|
100
|
-
const env = await this
|
|
158
|
+
const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
|
|
101
159
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
102
160
|
// envelope; say so rather than "failed: undefined".
|
|
103
161
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
104
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this
|
|
162
|
+
throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
105
163
|
}
|
|
106
164
|
if (!env.success) {
|
|
107
165
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
108
166
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
109
|
-
|
|
167
|
+
const type = env.error?.__type;
|
|
168
|
+
throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
|
|
110
169
|
}
|
|
111
170
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
112
171
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -141,10 +200,14 @@ export class CkanClient {
|
|
|
141
200
|
* refused (CkanValidationError), never sent or dropped.
|
|
142
201
|
*/
|
|
143
202
|
async packageSearch(params = {}) {
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
203
|
+
assertParams(params);
|
|
204
|
+
assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
|
|
205
|
+
assertText("q", params.q);
|
|
206
|
+
if (params.fq !== undefined)
|
|
207
|
+
assertValid("fq", params.fq, textListProblem);
|
|
208
|
+
assertText("sort", params.sort);
|
|
209
|
+
if (params.facet_field !== undefined)
|
|
210
|
+
assertValid("facet_field", params.facet_field, textListProblem);
|
|
148
211
|
assertCount("rows", params.rows);
|
|
149
212
|
assertCount("start", params.start);
|
|
150
213
|
if (params.facet_limit !== undefined)
|
|
@@ -180,6 +243,8 @@ export class CkanClient {
|
|
|
180
243
|
}
|
|
181
244
|
/** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
|
|
182
245
|
async packageList(params = {}) {
|
|
246
|
+
assertParams(params);
|
|
247
|
+
assertKeys("packageList", "package_list", params, PACKAGE_LIST_KEYS);
|
|
183
248
|
assertLimit(params.limit);
|
|
184
249
|
assertCount("offset", params.offset);
|
|
185
250
|
return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
|
|
@@ -196,7 +261,10 @@ export class CkanClient {
|
|
|
196
261
|
return this.groupOrOrgList("group_list", params);
|
|
197
262
|
}
|
|
198
263
|
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
199
|
-
tagList(params = {}) {
|
|
264
|
+
async tagList(params = {}) {
|
|
265
|
+
assertParams(params);
|
|
266
|
+
assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
|
|
267
|
+
assertText("query", params.query);
|
|
200
268
|
return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
|
|
201
269
|
}
|
|
202
270
|
/** The licences this portal offers. */
|
|
@@ -214,8 +282,22 @@ export class CkanClient {
|
|
|
214
282
|
* some entries), so only an empty page ends the list. Entries are deduplicated
|
|
215
283
|
* (by `id`), and a page that adds nothing new ends the loop too, so a server
|
|
216
284
|
* that ignores `offset` cannot keep it going.
|
|
285
|
+
*
|
|
286
|
+
* Berlin counts hidden entries in its `limit`/`offset` window but leaves them
|
|
287
|
+
* out of the answer, so a window smaller than a full page (the last one before
|
|
288
|
+
* `limit` is reached) can come back empty although more entries follow. Such a
|
|
289
|
+
* miss is asked again at the same offset with a full page before it ends the list.
|
|
290
|
+
*
|
|
291
|
+
* Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
|
|
292
|
+
* `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
|
|
293
|
+
* keep returning new entries) the call fails with a CkanParseError.
|
|
217
294
|
*/
|
|
218
295
|
async groupOrOrgList(action, params) {
|
|
296
|
+
assertParams(params);
|
|
297
|
+
assertKeys(action === "group_list" ? "groupList" : "organizationList", action, params, GROUP_LIST_KEYS);
|
|
298
|
+
if (params.all_fields !== undefined && typeof params.all_fields !== "boolean") {
|
|
299
|
+
throw new CkanValidationError("Invalid all_fields: Expected a boolean.");
|
|
300
|
+
}
|
|
219
301
|
assertLimit(params.limit);
|
|
220
302
|
// Before the first request, and before the all_fields pager steps on from it.
|
|
221
303
|
assertCount("offset", params.offset);
|
|
@@ -226,12 +308,25 @@ export class CkanClient {
|
|
|
226
308
|
let offset = params.offset ?? 0;
|
|
227
309
|
const seen = new Set();
|
|
228
310
|
const out = [];
|
|
311
|
+
let pages = 0;
|
|
312
|
+
// Set after a short window came back empty (or with nothing new): from then on
|
|
313
|
+
// every page is a full one.
|
|
314
|
+
let fullPages = false;
|
|
229
315
|
while (out.length < wanted) {
|
|
316
|
+
if (pages >= MAX_ALL_FIELDS_PAGES) {
|
|
317
|
+
throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
|
|
318
|
+
`reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
|
|
319
|
+
`Pass a limit (--limit) to fetch part of it.`);
|
|
320
|
+
}
|
|
321
|
+
if (pages > 0)
|
|
322
|
+
await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
|
|
323
|
+
const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
|
|
230
324
|
const page = await this.typed(action, {
|
|
231
325
|
all_fields: true,
|
|
232
|
-
limit
|
|
326
|
+
limit,
|
|
233
327
|
offset: offset === 0 ? undefined : offset,
|
|
234
328
|
}, Array.isArray, "an array");
|
|
329
|
+
pages += 1;
|
|
235
330
|
let added = 0;
|
|
236
331
|
for (const entry of page) {
|
|
237
332
|
const key = entryKey(entry);
|
|
@@ -241,8 +336,15 @@ export class CkanClient {
|
|
|
241
336
|
out.push(entry);
|
|
242
337
|
added += 1;
|
|
243
338
|
}
|
|
244
|
-
if (page.length === 0 || added === 0)
|
|
339
|
+
if (page.length === 0 || added === 0) {
|
|
340
|
+
// A short window may have held only hidden entries: ask once more, same
|
|
341
|
+
// offset, for a full page before taking it as the end.
|
|
342
|
+
if (limit < ALL_FIELDS_PAGE) {
|
|
343
|
+
fullPages = true;
|
|
344
|
+
continue;
|
|
345
|
+
}
|
|
245
346
|
break;
|
|
347
|
+
}
|
|
246
348
|
offset += page.length;
|
|
247
349
|
}
|
|
248
350
|
return out.slice(0, wanted);
|
|
@@ -252,8 +354,8 @@ export class CkanClient {
|
|
|
252
354
|
* CKAN would answer with a 409 validation error instead.
|
|
253
355
|
*/
|
|
254
356
|
async show(action, id) {
|
|
255
|
-
if (id.trim() === "")
|
|
256
|
-
throw new
|
|
357
|
+
if (typeof id !== "string" || id.trim() === "")
|
|
358
|
+
throw new CkanValidationError(`${action} needs an id or name.`);
|
|
257
359
|
return this.typed(action, { id }, isObject, "a JSON object");
|
|
258
360
|
}
|
|
259
361
|
}
|
|
@@ -18,21 +18,32 @@ export interface EngineOptions {
|
|
|
18
18
|
* not blank, no control characters but tab, nothing above U+00FF.
|
|
19
19
|
*/
|
|
20
20
|
userAgent?: string;
|
|
21
|
-
/**
|
|
21
|
+
/**
|
|
22
|
+
* Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
|
|
23
|
+
* It covers the whole response, body included, and the engine enforces it for every
|
|
24
|
+
* transport (the request also gets an AbortSignal that fires at the deadline).
|
|
25
|
+
*/
|
|
22
26
|
timeoutMs?: number;
|
|
23
27
|
/**
|
|
24
|
-
* Number of automatic retries for transient (429/503) responses
|
|
25
|
-
*
|
|
26
|
-
*
|
|
28
|
+
* Number of automatic retries for transient (429/503) responses; a reset or other
|
|
29
|
+
* network error is not retried. Each waits `retryDelayMs * attempt`, or the
|
|
30
|
+
* response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
|
|
31
|
+
* one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
|
|
27
32
|
*/
|
|
28
33
|
maxRetries?: number;
|
|
29
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* Base backoff between retries in milliseconds (grows linearly), 0 to
|
|
36
|
+
* `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
|
|
37
|
+
* `Retry-After`: the header can lengthen a wait, never shorten it.
|
|
38
|
+
*/
|
|
30
39
|
retryDelayMs?: number;
|
|
31
40
|
/** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
|
|
32
41
|
maxRedirects?: number;
|
|
33
42
|
/**
|
|
34
43
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
35
44
|
* from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
|
|
45
|
+
* Enforced by the engine for every transport: the built-in one aborts early, a
|
|
46
|
+
* custom one's body is checked when it arrives.
|
|
36
47
|
*
|
|
37
48
|
* Every numeric option must be a non-negative safe integer within its range;
|
|
38
49
|
* the constructor throws CkanValidationError otherwise (a negative or NaN
|
|
@@ -54,8 +65,8 @@ export declare function assertHeaderValue(name: string, value: string): string;
|
|
|
54
65
|
/**
|
|
55
66
|
* Longest `Retry-After` the engine waits out before retrying a 429/503. When the
|
|
56
67
|
* server asks for longer, the engine does not retry at all and surfaces the error at
|
|
57
|
-
* once: retrying early would only land inside the window
|
|
58
|
-
* out, and a hostile value must not stall the CLI.
|
|
68
|
+
* once, naming the requested wait: retrying early would only land inside the window
|
|
69
|
+
* the server asked us to wait out, and a hostile value must not stall the CLI.
|
|
59
70
|
*/
|
|
60
71
|
export declare const MAX_RETRY_AFTER_MS = 30000;
|
|
61
72
|
/**
|
|
@@ -105,8 +116,21 @@ export declare function sanitizeServerText(text: string): string;
|
|
|
105
116
|
* default transport still re-checks the scheme on every hop.
|
|
106
117
|
*/
|
|
107
118
|
export declare function validateBaseUrl(raw: string): string;
|
|
119
|
+
/**
|
|
120
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
121
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
122
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
123
|
+
* where nothing leaves the machine.
|
|
124
|
+
*
|
|
125
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
126
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
127
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
128
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
129
|
+
* (once per run, before the first request).
|
|
130
|
+
*/
|
|
131
|
+
export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
|
|
108
132
|
export declare class RequestEngine {
|
|
109
|
-
private
|
|
133
|
+
#private;
|
|
110
134
|
private readonly transport;
|
|
111
135
|
private readonly userAgent;
|
|
112
136
|
private readonly timeoutMs;
|
|
@@ -116,8 +140,37 @@ export declare class RequestEngine {
|
|
|
116
140
|
private readonly maxResponseBytes;
|
|
117
141
|
private readonly sleep;
|
|
118
142
|
constructor(options?: EngineOptions);
|
|
119
|
-
/**
|
|
143
|
+
/**
|
|
144
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
145
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
146
|
+
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
147
|
+
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
148
|
+
* messages built from server text.
|
|
149
|
+
*/
|
|
150
|
+
scrub(text: string): string;
|
|
151
|
+
/**
|
|
152
|
+
* A failure as the `cause` of the error the engine raises: the original when its
|
|
153
|
+
* text carries no credentials, otherwise a copy with them scrubbed (message, `code`
|
|
154
|
+
* and the cause chain kept), so logging the error with its causes can't reveal the
|
|
155
|
+
* base URL's password.
|
|
156
|
+
*/
|
|
157
|
+
private scrubCause;
|
|
158
|
+
/**
|
|
159
|
+
* Build a fully-qualified URL from a path and optional query parameters. It keeps
|
|
160
|
+
* the base URL's userinfo; `request()` leaves it out and sends it as an
|
|
161
|
+
* Authorization header instead (see basicAuthorization).
|
|
162
|
+
*/
|
|
120
163
|
buildUrl(path: string, query?: QueryParams): string;
|
|
164
|
+
/** buildUrl, with or without the base URL's userinfo. */
|
|
165
|
+
private composeUrl;
|
|
166
|
+
/**
|
|
167
|
+
* Call the transport under the overall deadline (`timeoutMs`): the request gets an
|
|
168
|
+
* AbortSignal that fires at the deadline, and the call rejects then whether the
|
|
169
|
+
* transport stops or not — a custom transport (fetch, a node:http wrapper) that
|
|
170
|
+
* ignores `timeoutMs` can't hang the caller (or `checkPortals`). A synchronous throw
|
|
171
|
+
* becomes a rejection.
|
|
172
|
+
*/
|
|
173
|
+
private callTransport;
|
|
121
174
|
/** Perform a request with Accept negotiation and transient-error retries. */
|
|
122
175
|
request(method: string, path: string, options?: {
|
|
123
176
|
query?: QueryParams;
|