@maschinenlesbar.org/ckan-cli 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -7
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +80 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +289 -41
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +16 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +33 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +64 -9
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +169 -18
- package/dist/src/cli/commands/catalogue.js +12 -11
- package/dist/src/cli/commands/portal.js +28 -20
- package/dist/src/cli/io.d.ts +16 -0
- package/dist/src/cli/io.js +55 -1
- package/dist/src/cli/output.d.ts +26 -0
- package/dist/src/cli/output.js +151 -0
- package/dist/src/cli/run.js +76 -6
- package/dist/src/cli/shared.d.ts +14 -25
- package/dist/src/cli/shared.js +40 -102
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -155,8 +155,14 @@ action <name> [--param key=value …] call any CKAN action (generic)
|
|
|
155
155
|
> **Several `--fq` filters** are sent as CKAN's `fq_list`, never as a repeated
|
|
156
156
|
> `fq` key, which CKAN answers with HTTP 409.
|
|
157
157
|
|
|
158
|
-
> **
|
|
159
|
-
>
|
|
158
|
+
> **Values go out canonical, or are refused.** The search text, `--fq`, `tags --query`
|
|
159
|
+
> and `--param` values are sent trimmed and in composed Unicode (NFC); ids, `--sort`,
|
|
160
|
+
> `--facet` and `--param` names trimmed. A blank value (`ckan search ""`, `--fq ""`,
|
|
161
|
+
> `tags --query ""`, `action … --param q=`, or one of only invisible characters) exits 1
|
|
162
|
+
> before any request, instead of silently running an unfiltered search; so does a value
|
|
163
|
+
> with a control or invisible character (a zero-width space pasted from a web page), shown
|
|
164
|
+
> escaped (`"el\u200bbe"`), a `--sort` that is not `<field> asc|desc[, …]`, a `--facet`
|
|
165
|
+
> that is not a field name, and an id with a `/`.
|
|
160
166
|
> So does a forgotten value (`--fq --rows 5`): a value of `--fq`, `--facet`,
|
|
161
167
|
> `--sort` or `tags --query` cannot start with `--`. And so does a single-value
|
|
162
168
|
> option given twice (`--rows 5 --rows 50`, a second `--base-url`): only `--fq`,
|
|
@@ -222,8 +228,10 @@ Found by running the same commands against many portals:
|
|
|
222
228
|
- **Solr syntax errors** come back as HTTP 409; `ckan` prints Solr's own reason,
|
|
223
229
|
e.g. `Search Error: Cannot parse 'title:(': Encountered "<EOF>" …`.
|
|
224
230
|
- **A site that is not (or no longer) a CKAN** answers with a 404, an HTML page or
|
|
225
|
-
a redirect loop; `ckan` says which, e.g.
|
|
226
|
-
`
|
|
231
|
+
a redirect loop; `ckan` says which, naming the request, e.g.
|
|
232
|
+
`GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "text/html").`
|
|
233
|
+
Every error about an answer starts with the request (`GET <url>: …`), CKAN's own error
|
|
234
|
+
answer sent with HTTP 200 too, so a run over many portals tells which one failed.
|
|
227
235
|
|
|
228
236
|
## Output & scripting
|
|
229
237
|
|
|
@@ -234,7 +242,11 @@ prints one line. Every global option works before or after the command.
|
|
|
234
242
|
Each line on stderr is a **log record**: a timestamp (UTC), a level (`ERROR`, `WARN`,
|
|
235
243
|
`INFO`) and a topic, the program and the area it comes from (`ckan.cli` for usage
|
|
236
244
|
errors, `ckan.api` for the portal's answers, a malformed one included, `ckan.http` for the connection,
|
|
237
|
-
`ckan.output` when stdout can't be written
|
|
245
|
+
`ckan.output` when stdout can't be written, a redirected file cut short by a full disk
|
|
246
|
+
included, or an answer can't be printed). Text in an
|
|
247
|
+
answer is never changed silently: bytes that are not valid in its charset and half
|
|
248
|
+
characters (lone surrogates) are replaced with U+FFFD, so the output stays readable for
|
|
249
|
+
`jq`, and a `WARN` record of `ckan.api` says how many. By default
|
|
238
250
|
it is written log4j style; `--log-format jsonl` writes one JSON object per line instead.
|
|
239
251
|
A record is always one line: a line break, a control character or a bidi control in a
|
|
240
252
|
message (a server's text, a value you typed) is written as an escape (`\n`, `\u001b`,
|
|
@@ -279,9 +291,11 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
|
|
|
279
291
|
> **Numbers pass through JavaScript.** The output is the parsed and re-serialised
|
|
280
292
|
> JSON. A number that would not print as the server wrote it (an integer above 2⁵³,
|
|
281
293
|
> 9,007,199,254,740,992, that loses its last digits, a number beyond the double range,
|
|
282
|
-
> `1e400`,
|
|
294
|
+
> `1e400`, one too small, `1e-400`, or one below the normal double range that a double
|
|
295
|
+
> would change, `3e-324`) is an error (exit 1, `the number … cannot be
|
|
283
296
|
> represented exactly`), never a changed value; one the server wrote in a double's
|
|
284
|
-
> shortest form (`1E23`) prints as written.
|
|
297
|
+
> shortest form (`1E23`) prints as written. Solr's `_version_` (`--param fl=_version_`) is
|
|
298
|
+
> always printed as a string with its exact digits, as Hamburg and GovData send it. CKAN metadata rarely holds such numbers; for
|
|
285
299
|
> an exact copy of the raw answer, use `curl` on `<site>/api/3/action/<name>`.
|
|
286
300
|
|
|
287
301
|
## Global options
|
|
@@ -27,12 +27,20 @@ export declare class CkanClient {
|
|
|
27
27
|
constructor(options?: EngineOptions);
|
|
28
28
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
29
29
|
action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
|
|
30
|
+
/**
|
|
31
|
+
* Call an action: its unwrapped `result` and the request as error messages name it
|
|
32
|
+
* (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
|
|
33
|
+
* with that request.
|
|
34
|
+
*/
|
|
35
|
+
private call;
|
|
30
36
|
/**
|
|
31
37
|
* An action whose `result` must have a known top-level shape (never a deep
|
|
32
38
|
* schema): a broken or foreign answer becomes a CkanParseError naming the
|
|
33
|
-
*
|
|
39
|
+
* request, not a TypeError further down (`page is not iterable`).
|
|
34
40
|
*/
|
|
35
41
|
private typed;
|
|
42
|
+
/** `typed`, with the request it made (for the `all_fields` pager's message). */
|
|
43
|
+
private typedCall;
|
|
36
44
|
/** Site title, CKAN version and enabled extensions of the portal. */
|
|
37
45
|
status(): Promise<Status>;
|
|
38
46
|
/**
|
|
@@ -44,8 +52,10 @@ export declare class CkanClient {
|
|
|
44
52
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
45
53
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
46
54
|
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
47
|
-
* refused (CkanValidationError), never sent or dropped. The search text `q`
|
|
48
|
-
* out trimmed and in NFC (`normalizeText`);
|
|
55
|
+
* refused (CkanValidationError), never sent or dropped. The search text `q` and each
|
|
56
|
+
* filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
|
|
57
|
+
* expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
|
|
58
|
+
* a Solr field name) as tokens (`normalizeToken`).
|
|
49
59
|
*/
|
|
50
60
|
packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
|
|
51
61
|
/** A single dataset by id or name. */
|
|
@@ -92,8 +102,9 @@ export declare class CkanClient {
|
|
|
92
102
|
*/
|
|
93
103
|
private groupOrOrgList;
|
|
94
104
|
/**
|
|
95
|
-
* A `*_show` call.
|
|
96
|
-
*
|
|
105
|
+
* A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
|
|
106
|
+
* invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
|
|
107
|
+
* blank one and CKAN would answer with a 409 validation error instead.
|
|
97
108
|
*/
|
|
98
109
|
private show;
|
|
99
110
|
}
|
|
@@ -1,9 +1,15 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
|
-
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
-
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
-
import { assertValid,
|
|
3
|
+
import { bigIntegerKeysOption, DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
+
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, quoteValue, redactUrl } from "./errors.js";
|
|
5
|
+
import { assertValid, countProblem, facetLimitProblem, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, queryValueProblem, sortProblem, textProblem, tokenProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
|
+
/**
|
|
8
|
+
* The members every CkanClient declares as big integers (`EngineOptions.bigIntegerKeys`): Solr's
|
|
9
|
+
* `_version_`, which `package_search` returns for `fl=_version_` (Hamburg and GovData send it as
|
|
10
|
+
* a string; a number would be past 2^53).
|
|
11
|
+
*/
|
|
12
|
+
const BIG_INTEGER_KEYS = ["_version_"];
|
|
7
13
|
/**
|
|
8
14
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
9
15
|
* the path-traversal hole (`../../..` escaping `/api/3/action/`) and the
|
|
@@ -37,26 +43,40 @@ function entryKey(entry) {
|
|
|
37
43
|
return JSON.stringify(entry);
|
|
38
44
|
}
|
|
39
45
|
/**
|
|
40
|
-
* Drop undefined values so only the parameters the caller actually set are sent,
|
|
41
|
-
*
|
|
42
|
-
* (
|
|
43
|
-
*
|
|
46
|
+
* Drop undefined values so only the parameters the caller actually set are sent, and send
|
|
47
|
+
* each name as a token (`normalizeToken`: trimmed, NFC) and each string value as text
|
|
48
|
+
* (`normalizeText`). A blank or invisible name, a blank value, a value with a control or
|
|
49
|
+
* format character, and two names that are the same once trimmed (`q` and ` q`) are refused
|
|
50
|
+
* (CkanValidationError): CKAN reads an empty parameter as "not given", so a blank filter such
|
|
51
|
+
* as `tagList({ query: "" })` would silently return everything.
|
|
44
52
|
*/
|
|
45
53
|
function prune(params) {
|
|
46
54
|
assertParams(params);
|
|
47
55
|
// A null-prototype object, so a `__proto__` key is kept as a parameter instead
|
|
48
56
|
// of setting the prototype (and being lost).
|
|
49
57
|
const out = Object.create(null);
|
|
58
|
+
const given = new Map();
|
|
50
59
|
for (const [k, v] of Object.entries(params)) {
|
|
51
60
|
if (v === undefined)
|
|
52
61
|
continue;
|
|
53
|
-
assertValid("parameter name", k,
|
|
62
|
+
const name = normalizeToken(assertValid("parameter name", k, tokenProblem));
|
|
63
|
+
const earlier = given.get(name);
|
|
64
|
+
if (earlier !== undefined) {
|
|
65
|
+
throw new CkanValidationError(`Invalid params: ${quoteValue(name)} is given twice (as ${quoteValue(earlier)} and ${quoteValue(k)}).`);
|
|
66
|
+
}
|
|
67
|
+
given.set(name, k);
|
|
54
68
|
// A value the query string can't carry (an object, NaN, a function) is refused,
|
|
55
69
|
// not sent as "[object Object]" or "NaN".
|
|
56
|
-
out[
|
|
70
|
+
out[name] = textValue(assertValid(k, v, queryValueProblem));
|
|
57
71
|
}
|
|
58
72
|
return out;
|
|
59
73
|
}
|
|
74
|
+
/** A query value with each string in it sent as text (`normalizeText`); other values as they are. */
|
|
75
|
+
function textValue(value) {
|
|
76
|
+
if (typeof value === "string")
|
|
77
|
+
return normalizeText(value);
|
|
78
|
+
return Array.isArray(value) ? value.map((v) => (typeof v === "string" ? normalizeText(v) : v)) : value;
|
|
79
|
+
}
|
|
60
80
|
/**
|
|
61
81
|
* A method's parameter object: a plain object (or undefined, for the defaults).
|
|
62
82
|
* `null`, a string or an array would otherwise fail as a raw TypeError.
|
|
@@ -77,7 +97,7 @@ function assertParams(params) {
|
|
|
77
97
|
function assertKeys(method, action, params, known) {
|
|
78
98
|
for (const key of Object.keys(params)) {
|
|
79
99
|
if (!known.includes(key)) {
|
|
80
|
-
throw new CkanValidationError(`Invalid ${method} parameter ${
|
|
100
|
+
throw new CkanValidationError(`Invalid ${method} parameter ${quoteValue(key)}: not a parameter of ${method}. ` +
|
|
81
101
|
`Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
|
|
82
102
|
}
|
|
83
103
|
}
|
|
@@ -86,10 +106,19 @@ const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_l
|
|
|
86
106
|
const PACKAGE_LIST_KEYS = ["limit", "offset"];
|
|
87
107
|
const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
|
|
88
108
|
const TAG_LIST_KEYS = ["query"];
|
|
89
|
-
/** Check an optional
|
|
90
|
-
function
|
|
109
|
+
/** Check an optional parameter with `problem`; `undefined` means "not given". */
|
|
110
|
+
function assertOptional(name, value, problem) {
|
|
91
111
|
if (value !== undefined)
|
|
92
|
-
assertValid(name, value,
|
|
112
|
+
assertValid(name, value, problem);
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Check an optional list parameter (`fq`, `facet_field`): each entry with `itemProblem`, a blank
|
|
116
|
+
* entry refused, never dropped. An empty list means "not given", as before.
|
|
117
|
+
*/
|
|
118
|
+
function assertList(name, value, itemProblem) {
|
|
119
|
+
if (value === undefined || (Array.isArray(value) && value.length === 0))
|
|
120
|
+
return;
|
|
121
|
+
assertValid(name, value, (v) => listProblem(v, itemProblem));
|
|
93
122
|
}
|
|
94
123
|
/** A JSON object (not null, not an array). */
|
|
95
124
|
function isObject(value) {
|
|
@@ -102,9 +131,9 @@ function isSearchResult(value) {
|
|
|
102
131
|
const count = value["count"];
|
|
103
132
|
return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 && Array.isArray(value["results"]);
|
|
104
133
|
}
|
|
105
|
-
/** The error for a `result` that does not have the shape the caller relies on. */
|
|
106
|
-
function shapeError(
|
|
107
|
-
return new CkanParseError(
|
|
134
|
+
/** The error for a `result` that does not have the shape the caller relies on, naming the request (`GET <url>`). */
|
|
135
|
+
function shapeError(request, expected) {
|
|
136
|
+
return new CkanParseError(`${request}: unexpected response shape: expected ${expected}.`);
|
|
108
137
|
}
|
|
109
138
|
/**
|
|
110
139
|
* Check a `*_list` limit. CKAN reads `limit=0` as "no limit" (a whole catalogue,
|
|
@@ -146,43 +175,61 @@ export class CkanClient {
|
|
|
146
175
|
#sleep;
|
|
147
176
|
constructor(options = {}) {
|
|
148
177
|
this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
|
|
149
|
-
// The engine checks every option, `sleep` included, before the client uses it.
|
|
150
|
-
|
|
178
|
+
// The engine checks every option, `sleep` included, before the client uses it. Solr's
|
|
179
|
+
// `_version_` (≈1.88e18, past 2^53) is always declared a big integer: a portal that sends it
|
|
180
|
+
// as a number gets the string Hamburg and GovData send, never a refused answer.
|
|
181
|
+
this.#engine = new RequestEngine({
|
|
182
|
+
...options,
|
|
183
|
+
baseUrl: this.#site,
|
|
184
|
+
bigIntegerKeys: new Set([...bigIntegerKeysOption(options.bigIntegerKeys), ...BIG_INTEGER_KEYS]),
|
|
185
|
+
});
|
|
151
186
|
this.#sleep = options.sleep ?? realSleep;
|
|
152
187
|
}
|
|
153
188
|
/** Call any CKAN action by name and return its unwrapped `result`. */
|
|
154
189
|
async action(name, params = {}) {
|
|
190
|
+
return (await this.call(name, params)).result;
|
|
191
|
+
}
|
|
192
|
+
/**
|
|
193
|
+
* Call an action: its unwrapped `result` and the request as error messages name it
|
|
194
|
+
* (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
|
|
195
|
+
* with that request.
|
|
196
|
+
*/
|
|
197
|
+
async call(name, params) {
|
|
155
198
|
if (typeof name !== "string" || !ACTION_NAME.test(name)) {
|
|
156
|
-
throw new CkanValidationError(`Invalid CKAN action name:
|
|
199
|
+
throw new CkanValidationError(`Invalid CKAN action name: ${quoteValue(typeof name === "string" ? name : String(name))}`);
|
|
157
200
|
}
|
|
158
|
-
const env = await this.#engine.
|
|
201
|
+
const { value: env, url, request } = await this.#engine.getAnswer(`${ACTION}/${name}`, prune(params));
|
|
159
202
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
160
203
|
// envelope; say so rather than "failed: undefined".
|
|
161
204
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
162
|
-
throw new CkanParseError(
|
|
205
|
+
throw new CkanParseError(`${request}: the answer is not a CKAN Action API response; is ${cutForMessage(redactUrl(this.#site))} a CKAN site?`);
|
|
163
206
|
}
|
|
164
207
|
if (!env.success) {
|
|
165
208
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
166
209
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
167
210
|
const type = env.error?.__type;
|
|
168
|
-
throw new CkanActionError(
|
|
211
|
+
throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined, url });
|
|
169
212
|
}
|
|
170
213
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
171
214
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
172
215
|
if (env.result === undefined)
|
|
173
|
-
throw shapeError(
|
|
174
|
-
return env.result;
|
|
216
|
+
throw shapeError(request, "a result in the envelope");
|
|
217
|
+
return { result: env.result, request };
|
|
175
218
|
}
|
|
176
219
|
/**
|
|
177
220
|
* An action whose `result` must have a known top-level shape (never a deep
|
|
178
221
|
* schema): a broken or foreign answer becomes a CkanParseError naming the
|
|
179
|
-
*
|
|
222
|
+
* request, not a TypeError further down (`page is not iterable`).
|
|
180
223
|
*/
|
|
181
224
|
async typed(name, params, ok, expected) {
|
|
182
|
-
|
|
225
|
+
return (await this.typedCall(name, params, ok, expected)).result;
|
|
226
|
+
}
|
|
227
|
+
/** `typed`, with the request it made (for the `all_fields` pager's message). */
|
|
228
|
+
async typedCall(name, params, ok, expected) {
|
|
229
|
+
const { result, request } = await this.call(name, params);
|
|
183
230
|
if (!ok(result))
|
|
184
|
-
throw shapeError(
|
|
185
|
-
return result;
|
|
231
|
+
throw shapeError(request, expected);
|
|
232
|
+
return { result: result, request };
|
|
186
233
|
}
|
|
187
234
|
/** Site title, CKAN version and enabled extensions of the portal. */
|
|
188
235
|
status() {
|
|
@@ -197,31 +244,31 @@ export class CkanClient {
|
|
|
197
244
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
198
245
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
199
246
|
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
200
|
-
* refused (CkanValidationError), never sent or dropped. The search text `q`
|
|
201
|
-
* out trimmed and in NFC (`normalizeText`);
|
|
247
|
+
* refused (CkanValidationError), never sent or dropped. The search text `q` and each
|
|
248
|
+
* filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
|
|
249
|
+
* expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
|
|
250
|
+
* a Solr field name) as tokens (`normalizeToken`).
|
|
202
251
|
*/
|
|
203
252
|
async packageSearch(params = {}) {
|
|
204
253
|
assertParams(params);
|
|
205
254
|
assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
if (params.facet_field !== undefined)
|
|
211
|
-
assertValid("facet_field", params.facet_field, textListProblem);
|
|
255
|
+
assertOptional("q", params.q, textProblem);
|
|
256
|
+
assertList("fq", params.fq, textProblem);
|
|
257
|
+
assertOptional("sort", params.sort, sortProblem);
|
|
258
|
+
assertList("facet_field", params.facet_field, fieldProblem);
|
|
212
259
|
assertCount("rows", params.rows);
|
|
213
260
|
assertCount("start", params.start);
|
|
214
261
|
if (params.facet_limit !== undefined)
|
|
215
262
|
assertValid("facet_limit", params.facet_limit, facetLimitProblem);
|
|
216
|
-
const fq = params.fq ?? [];
|
|
217
|
-
const facetFields = params.facet_field ?? [];
|
|
263
|
+
const fq = (params.fq ?? []).map(normalizeText);
|
|
264
|
+
const facetFields = (params.facet_field ?? []).map(normalizeToken);
|
|
218
265
|
return this.typed("package_search", {
|
|
219
266
|
q: params.q === undefined ? undefined : normalizeText(params.q),
|
|
220
267
|
fq: fq.length === 1 ? fq[0] : undefined,
|
|
221
268
|
fq_list: fq.length > 1 ? fq : undefined,
|
|
222
269
|
rows: params.rows,
|
|
223
270
|
start: params.start,
|
|
224
|
-
sort: params.sort,
|
|
271
|
+
sort: params.sort === undefined ? undefined : normalizeToken(params.sort),
|
|
225
272
|
"facet.field": facetFields.length > 0 ? JSON.stringify(facetFields) : undefined,
|
|
226
273
|
"facet.limit": params.facet_limit,
|
|
227
274
|
}, isSearchResult, "an object with a numeric count and a results array");
|
|
@@ -265,7 +312,7 @@ export class CkanClient {
|
|
|
265
312
|
async tagList(params = {}) {
|
|
266
313
|
assertParams(params);
|
|
267
314
|
assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
|
|
268
|
-
|
|
315
|
+
assertOptional("query", params.query, textProblem);
|
|
269
316
|
const query = params.query === undefined ? undefined : normalizeText(params.query);
|
|
270
317
|
return this.typed("tag_list", { query }, Array.isArray, "an array");
|
|
271
318
|
}
|
|
@@ -314,20 +361,24 @@ export class CkanClient {
|
|
|
314
361
|
// Set after a short window came back empty (or with nothing new): from then on
|
|
315
362
|
// every page is a full one.
|
|
316
363
|
let fullPages = false;
|
|
364
|
+
/** The last page's request, which the stop message names. */
|
|
365
|
+
let request = "";
|
|
317
366
|
while (out.length < wanted) {
|
|
318
367
|
if (pages >= MAX_ALL_FIELDS_PAGES) {
|
|
319
|
-
throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
|
|
368
|
+
throw new CkanParseError(`${request}: ${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
|
|
320
369
|
`reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
|
|
321
370
|
`Pass a limit (--limit) to fetch part of it.`);
|
|
322
371
|
}
|
|
323
372
|
if (pages > 0)
|
|
324
373
|
await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
|
|
325
374
|
const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
|
|
326
|
-
const
|
|
375
|
+
const answer = await this.typedCall(action, {
|
|
327
376
|
all_fields: true,
|
|
328
377
|
limit,
|
|
329
378
|
offset: offset === 0 ? undefined : offset,
|
|
330
379
|
}, Array.isArray, "an array");
|
|
380
|
+
const page = answer.result;
|
|
381
|
+
request = answer.request;
|
|
331
382
|
pages += 1;
|
|
332
383
|
let added = 0;
|
|
333
384
|
for (const entry of page) {
|
|
@@ -352,12 +403,12 @@ export class CkanClient {
|
|
|
352
403
|
return out.slice(0, wanted);
|
|
353
404
|
}
|
|
354
405
|
/**
|
|
355
|
-
* A `*_show` call.
|
|
356
|
-
*
|
|
406
|
+
* A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
|
|
407
|
+
* invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
|
|
408
|
+
* blank one and CKAN would answer with a 409 validation error instead.
|
|
357
409
|
*/
|
|
358
410
|
async show(action, id) {
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
return this.typed(action, { id }, isObject, "a JSON object");
|
|
411
|
+
assertValid("id", id, idProblem);
|
|
412
|
+
return this.typed(action, { id: normalizeToken(id) }, isObject, "a JSON object");
|
|
362
413
|
}
|
|
363
414
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type Transport } from "./http.js";
|
|
2
2
|
import { type QueryParams } from "./query.js";
|
|
3
|
+
export { quoteValue } from "./errors.js";
|
|
3
4
|
export declare const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
4
5
|
export interface RawResponse {
|
|
5
6
|
data: Buffer;
|
|
@@ -57,6 +58,26 @@ export interface EngineOptions {
|
|
|
57
58
|
* and reset connection; never when there is no retry. A throw is swallowed.
|
|
58
59
|
*/
|
|
59
60
|
onRetry?: ((event: RetryEvent) => void) | undefined;
|
|
61
|
+
/**
|
|
62
|
+
* Called with a one-line warning about an answer the engine delivered changed: characters
|
|
63
|
+
* that were not valid text — invalid bytes in the declared (or default UTF-8) charset, or a
|
|
64
|
+
* lone surrogate written as a JSON escape — replaced with U+FFFD. The message names the
|
|
65
|
+
* request and the count (`GET <url>: 2 characters of the answer were not valid text …`).
|
|
66
|
+
* Without it the replacement is silent. A throw is swallowed. The CLI logs it as a `WARN`
|
|
67
|
+
* record of `ckan.api`.
|
|
68
|
+
*/
|
|
69
|
+
onWarning?: ((message: string) => void) | undefined;
|
|
70
|
+
/**
|
|
71
|
+
* Member names whose number values may exceed 2^53 (a Solr `_version_`, a 64-bit id). The value
|
|
72
|
+
* of a declared member is delivered as its exact literal, as written, as a JSON string (`"5"`),
|
|
73
|
+
* so its type never depends on its size. A declared member whose value is not a number (a
|
|
74
|
+
* string, `null`, an object) is left as it is. Every undeclared number keeps the rule: exact,
|
|
75
|
+
* or a CkanParseError. Matched by name anywhere in a JSON answer, an escaped name
|
|
76
|
+
* (`"\u005fversion_"`) included. Defaults to none in the engine (the CkanClient always adds
|
|
77
|
+
* `_version_`); anything but a set or list of non-empty strings makes the constructor throw a
|
|
78
|
+
* CkanValidationError. The set is copied.
|
|
79
|
+
*/
|
|
80
|
+
bigIntegerKeys?: ReadonlySet<string> | undefined;
|
|
60
81
|
}
|
|
61
82
|
/** What `EngineOptions.onRetry` is told about one retry. */
|
|
62
83
|
export interface RetryEvent {
|
|
@@ -147,6 +168,11 @@ export declare function validateBaseUrl(raw: string): string;
|
|
|
147
168
|
* (once per run, before the first request).
|
|
148
169
|
*/
|
|
149
170
|
export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
|
|
171
|
+
/**
|
|
172
|
+
* Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
|
|
173
|
+
* copied; anything else throws an CkanValidationError.
|
|
174
|
+
*/
|
|
175
|
+
export declare function bigIntegerKeysOption(value: ReadonlySet<string> | undefined): ReadonlySet<string>;
|
|
150
176
|
export declare class RequestEngine {
|
|
151
177
|
#private;
|
|
152
178
|
private readonly transport;
|
|
@@ -158,7 +184,12 @@ export declare class RequestEngine {
|
|
|
158
184
|
private readonly maxResponseBytes;
|
|
159
185
|
private readonly sleep;
|
|
160
186
|
private readonly onRetry;
|
|
187
|
+
private readonly onWarning;
|
|
188
|
+
/** `EngineOptions.bigIntegerKeys`, copied. */
|
|
189
|
+
readonly bigIntegerKeys: ReadonlySet<string>;
|
|
161
190
|
constructor(options?: EngineOptions);
|
|
191
|
+
/** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
|
|
192
|
+
private warn;
|
|
162
193
|
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
163
194
|
private backOff;
|
|
164
195
|
/**
|
|
@@ -197,7 +228,55 @@ export declare class RequestEngine {
|
|
|
197
228
|
query?: QueryParams;
|
|
198
229
|
accept: string;
|
|
199
230
|
}): Promise<RawResponse>;
|
|
200
|
-
/** Perform a GET expecting JSON and parse it into `T
|
|
231
|
+
/** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
|
|
201
232
|
getJson<T>(path: string, query?: QueryParams): Promise<T>;
|
|
233
|
+
/**
|
|
234
|
+
* Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
|
|
235
|
+
* userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
|
|
236
|
+
* MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
|
|
237
|
+
* (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
|
|
238
|
+
* one failed; the client prefixes its own envelope and shape errors with it too.
|
|
239
|
+
*
|
|
240
|
+
* The text is never changed silently: bytes that are not valid in the charset, and lone
|
|
241
|
+
* surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
|
|
242
|
+
* well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
|
|
243
|
+
* `onWarning` is told how many. One bad character never sinks a whole answer.
|
|
244
|
+
*/
|
|
245
|
+
getAnswer<T>(path: string, query?: QueryParams): Promise<{
|
|
246
|
+
value: T;
|
|
247
|
+
url: string;
|
|
248
|
+
request: string;
|
|
249
|
+
}>;
|
|
202
250
|
private toApiError;
|
|
203
251
|
}
|
|
252
|
+
/** What {@link scanNumbers} found in a JSON text. */
|
|
253
|
+
interface NumberScan {
|
|
254
|
+
/** The text with each declared member's number value quoted; the input itself when there was none. */
|
|
255
|
+
json: string;
|
|
256
|
+
/** The first undeclared number literal JSON.parse cannot carry exactly, or undefined. */
|
|
257
|
+
inexact: string | undefined;
|
|
258
|
+
}
|
|
259
|
+
/**
|
|
260
|
+
* One linear pass over `json` (the text of an answer) that skips strings and looks at every
|
|
261
|
+
* number literal:
|
|
262
|
+
* - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
|
|
263
|
+
* 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
|
|
264
|
+
* it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
|
|
265
|
+
* so a malformed one (`01`) still fails JSON.parse;
|
|
266
|
+
* - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
|
|
267
|
+
* cannot carry exactly is reported in `inexact`.
|
|
268
|
+
* Nothing is allocated per value: a member name is compared in place (only a name written with
|
|
269
|
+
* an escape is decoded), and the text is copied only when a declared member is found. No
|
|
270
|
+
* reviver, which would recurse per nesting level and overflow on deep data.
|
|
271
|
+
*/
|
|
272
|
+
export declare function scanNumbers(json: string, bigIntegerKeys?: ReadonlySet<string>): NumberScan;
|
|
273
|
+
/**
|
|
274
|
+
* The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
|
|
275
|
+
* undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
|
|
276
|
+
* 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
|
|
277
|
+
* `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
|
|
278
|
+
* (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
|
|
279
|
+
* `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
|
|
280
|
+
* precision, as JSON means it. {@link scanNumbers} without declared members.
|
|
281
|
+
*/
|
|
282
|
+
export declare function firstInexactNumber(json: string): string | undefined;
|