@maschinenlesbar.org/ckan-cli 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (24) hide show
  1. package/README.md +25 -9
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +17 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +102 -49
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +90 -11
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +337 -21
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +16 -1
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +33 -1
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +10 -7
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/types.d.ts +11 -11
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +73 -7
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +180 -16
  15. package/dist/src/cli/commands/catalogue.js +12 -11
  16. package/dist/src/cli/commands/portal.js +28 -20
  17. package/dist/src/cli/io.d.ts +16 -0
  18. package/dist/src/cli/io.js +55 -1
  19. package/dist/src/cli/output.d.ts +26 -0
  20. package/dist/src/cli/output.js +151 -0
  21. package/dist/src/cli/run.js +76 -6
  22. package/dist/src/cli/shared.d.ts +14 -25
  23. package/dist/src/cli/shared.js +40 -72
  24. package/package.json +1 -1
package/README.md CHANGED
@@ -155,8 +155,14 @@ action <name> [--param key=value …] call any CKAN action (generic)
155
155
  > **Several `--fq` filters** are sent as CKAN's `fq_list`, never as a repeated
156
156
  > `fq` key, which CKAN answers with HTTP 409.
157
157
 
158
- > **Blank values are refused.** `ckan search ""`, `--fq ""`, `tags --query ""` or
159
- > `action … --param q=` exit 1 before any request, instead of silently running an unfiltered search.
158
+ > **Values go out canonical, or are refused.** The search text, `--fq`, `tags --query`
159
+ > and `--param` values are sent trimmed and in composed Unicode (NFC); ids, `--sort`,
160
+ > `--facet` and `--param` names trimmed. A blank value (`ckan search ""`, `--fq ""`,
161
+ > `tags --query ""`, `action … --param q=`, or one of only invisible characters) exits 1
162
+ > before any request, instead of silently running an unfiltered search; so does a value
163
+ > with a control or invisible character (a zero-width space pasted from a web page), shown
164
+ > escaped (`"el\u200bbe"`), a `--sort` that is not `<field> asc|desc[, …]`, a `--facet`
165
+ > that is not a field name, and an id with a `/`.
160
166
  > So does a forgotten value (`--fq --rows 5`): a value of `--fq`, `--facet`,
161
167
  > `--sort` or `tags --query` cannot start with `--`. And so does a single-value
162
168
  > option given twice (`--rows 5 --rows 50`, a second `--base-url`): only `--fq`,
@@ -222,8 +228,10 @@ Found by running the same commands against many portals:
222
228
  - **Solr syntax errors** come back as HTTP 409; `ckan` prints Solr's own reason,
223
229
  e.g. `Search Error: Cannot parse 'title:(': Encountered "<EOF>" …`.
224
230
  - **A site that is not (or no longer) a CKAN** answers with a 404, an HTML page or
225
- a redirect loop; `ckan` says which, e.g.
226
- `Expected JSON from … but got text/html`.
231
+ a redirect loop; `ckan` says which, naming the request, e.g.
232
+ `GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "text/html").`
233
+ Every error about an answer starts with the request (`GET <url>: …`), CKAN's own error
234
+ answer sent with HTTP 200 too, so a run over many portals tells which one failed.
227
235
 
228
236
  ## Output & scripting
229
237
 
@@ -234,7 +242,11 @@ prints one line. Every global option works before or after the command.
234
242
  Each line on stderr is a **log record**: a timestamp (UTC), a level (`ERROR`, `WARN`,
235
243
  `INFO`) and a topic, the program and the area it comes from (`ckan.cli` for usage
236
244
  errors, `ckan.api` for the portal's answers, a malformed one included, `ckan.http` for the connection,
237
- `ckan.output` when stdout can't be written). By default
245
+ `ckan.output` when stdout can't be written, a redirected file cut short by a full disk
246
+ included, or an answer can't be printed). Text in an
247
+ answer is never changed silently: bytes that are not valid in its charset and half
248
+ characters (lone surrogates) are replaced with U+FFFD, so the output stays readable for
249
+ `jq`, and a `WARN` record of `ckan.api` says how many. By default
238
250
  it is written log4j style; `--log-format jsonl` writes one JSON object per line instead.
239
251
  A record is always one line: a line break, a control character or a bidi control in a
240
252
  message (a server's text, a value you typed) is written as an escape (`\n`, `\u001b`,
@@ -277,10 +289,14 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
277
289
  `Bad request - Action name not known: …`.
278
290
 
279
291
  > **Numbers pass through JavaScript.** The output is the parsed and re-serialised
280
- > JSON, so an integer above 2⁵³ (9,007,199,254,740,992) loses its last digits and a
281
- > number beyond the double range (`1e400`) prints as `null`. CKAN metadata rarely
282
- > holds such numbers; for an exact copy of the raw answer, use `curl` on
283
- > `<site>/api/3/action/<name>`.
292
+ > JSON. A number that would not print as the server wrote it (an integer above 2⁵³,
293
+ > 9,007,199,254,740,992, that loses its last digits, a number beyond the double range,
294
+ > `1e400`, one too small, `1e-400`, or one below the normal double range that a double
295
+ > would change, `3e-324`) is an error (exit 1, `the number … cannot be
296
+ > represented exactly`), never a changed value; one the server wrote in a double's
297
+ > shortest form (`1E23`) prints as written. Solr's `_version_` (`--param fl=_version_`) is
298
+ > always printed as a string with its exact digits, as Hamburg and GovData send it. CKAN metadata rarely holds such numbers; for
299
+ > an exact copy of the raw answer, use `curl` on `<site>/api/3/action/<name>`.
284
300
 
285
301
  ## Global options
286
302
 
@@ -27,12 +27,20 @@ export declare class CkanClient {
27
27
  constructor(options?: EngineOptions);
28
28
  /** Call any CKAN action by name and return its unwrapped `result`. */
29
29
  action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
30
+ /**
31
+ * Call an action: its unwrapped `result` and the request as error messages name it
32
+ * (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
33
+ * with that request.
34
+ */
35
+ private call;
30
36
  /**
31
37
  * An action whose `result` must have a known top-level shape (never a deep
32
38
  * schema): a broken or foreign answer becomes a CkanParseError naming the
33
- * action, not a TypeError further down (`page is not iterable`).
39
+ * request, not a TypeError further down (`page is not iterable`).
34
40
  */
35
41
  private typed;
42
+ /** `typed`, with the request it made (for the `all_fields` pager's message). */
43
+ private typedCall;
36
44
  /** Site title, CKAN version and enabled extensions of the portal. */
37
45
  status(): Promise<Status>;
38
46
  /**
@@ -44,7 +52,10 @@ export declare class CkanClient {
44
52
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
45
53
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
46
54
  * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
47
- * refused (CkanValidationError), never sent or dropped.
55
+ * refused (CkanValidationError), never sent or dropped. The search text `q` and each
56
+ * filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
57
+ * expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
58
+ * a Solr field name) as tokens (`normalizeToken`).
48
59
  */
49
60
  packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
50
61
  /** A single dataset by id or name. */
@@ -64,7 +75,7 @@ export declare class CkanClient {
64
75
  organizationList(params?: GroupListParams): Promise<JsonValue[]>;
65
76
  /** Group names, or full objects with `all_fields` (paged like organizationList). */
66
77
  groupList(params?: GroupListParams): Promise<JsonValue[]>;
67
- /** Tags, optionally only those containing a substring (a blank one is refused). */
78
+ /** Tags, optionally only those containing a substring (a blank one is refused; sent trimmed, in NFC). */
68
79
  tagList(params?: TagListParams): Promise<string[]>;
69
80
  /** The licences this portal offers. */
70
81
  licenseList(): Promise<License[]>;
@@ -91,8 +102,9 @@ export declare class CkanClient {
91
102
  */
92
103
  private groupOrOrgList;
93
104
  /**
94
- * A `*_show` call. A blank id is rejected up front: `prune` would drop it and
95
- * CKAN would answer with a 409 validation error instead.
105
+ * A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
106
+ * invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
107
+ * blank one and CKAN would answer with a 409 validation error instead.
96
108
  */
97
109
  private show;
98
110
  }
@@ -1,9 +1,15 @@
1
1
  // CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
2
2
  // Action API (`<site>/api/3/action`).
3
- import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
- import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
5
- import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
3
+ import { bigIntegerKeysOption, DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
+ import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, quoteValue, redactUrl } from "./errors.js";
5
+ import { assertValid, countProblem, facetLimitProblem, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, queryValueProblem, sortProblem, textProblem, tokenProblem, } from "./validate.js";
6
6
  const ACTION = "/api/3/action";
7
+ /**
8
+ * The members every CkanClient declares as big integers (`EngineOptions.bigIntegerKeys`): Solr's
9
+ * `_version_`, which `package_search` returns for `fl=_version_` (Hamburg and GovData send it as
10
+ * a string; a number would be past 2^53).
11
+ */
12
+ const BIG_INTEGER_KEYS = ["_version_"];
7
13
  /**
8
14
  * CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
9
15
  * the path-traversal hole (`../../..` escaping `/api/3/action/`) and the
@@ -37,26 +43,40 @@ function entryKey(entry) {
37
43
  return JSON.stringify(entry);
38
44
  }
39
45
  /**
40
- * Drop undefined values so only the parameters the caller actually set are sent,
41
- * and refuse a blank parameter name, a blank value or a list with a blank entry
42
- * (CkanValidationError). CKAN reads an empty parameter as "not given", so a blank
43
- * filter such as `tagList({ query: "" })` would silently return everything.
46
+ * Drop undefined values so only the parameters the caller actually set are sent, and send
47
+ * each name as a token (`normalizeToken`: trimmed, NFC) and each string value as text
48
+ * (`normalizeText`). A blank or invisible name, a blank value, a value with a control or
49
+ * format character, and two names that are the same once trimmed (`q` and ` q`) are refused
50
+ * (CkanValidationError): CKAN reads an empty parameter as "not given", so a blank filter such
51
+ * as `tagList({ query: "" })` would silently return everything.
44
52
  */
45
53
  function prune(params) {
46
54
  assertParams(params);
47
55
  // A null-prototype object, so a `__proto__` key is kept as a parameter instead
48
56
  // of setting the prototype (and being lost).
49
57
  const out = Object.create(null);
58
+ const given = new Map();
50
59
  for (const [k, v] of Object.entries(params)) {
51
60
  if (v === undefined)
52
61
  continue;
53
- assertValid("parameter name", k, blankProblem);
62
+ const name = normalizeToken(assertValid("parameter name", k, tokenProblem));
63
+ const earlier = given.get(name);
64
+ if (earlier !== undefined) {
65
+ throw new CkanValidationError(`Invalid params: ${quoteValue(name)} is given twice (as ${quoteValue(earlier)} and ${quoteValue(k)}).`);
66
+ }
67
+ given.set(name, k);
54
68
  // A value the query string can't carry (an object, NaN, a function) is refused,
55
69
  // not sent as "[object Object]" or "NaN".
56
- out[k] = assertValid(k, v, queryValueProblem);
70
+ out[name] = textValue(assertValid(k, v, queryValueProblem));
57
71
  }
58
72
  return out;
59
73
  }
74
+ /** A query value with each string in it sent as text (`normalizeText`); other values as they are. */
75
+ function textValue(value) {
76
+ if (typeof value === "string")
77
+ return normalizeText(value);
78
+ return Array.isArray(value) ? value.map((v) => (typeof v === "string" ? normalizeText(v) : v)) : value;
79
+ }
60
80
  /**
61
81
  * A method's parameter object: a plain object (or undefined, for the defaults).
62
82
  * `null`, a string or an array would otherwise fail as a raw TypeError.
@@ -77,7 +97,7 @@ function assertParams(params) {
77
97
  function assertKeys(method, action, params, known) {
78
98
  for (const key of Object.keys(params)) {
79
99
  if (!known.includes(key)) {
80
- throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
100
+ throw new CkanValidationError(`Invalid ${method} parameter ${quoteValue(key)}: not a parameter of ${method}. ` +
81
101
  `Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
82
102
  }
83
103
  }
@@ -86,10 +106,19 @@ const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_l
86
106
  const PACKAGE_LIST_KEYS = ["limit", "offset"];
87
107
  const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
88
108
  const TAG_LIST_KEYS = ["query"];
89
- /** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
90
- function assertText(name, value) {
109
+ /** Check an optional parameter with `problem`; `undefined` means "not given". */
110
+ function assertOptional(name, value, problem) {
91
111
  if (value !== undefined)
92
- assertValid(name, value, textProblem);
112
+ assertValid(name, value, problem);
113
+ }
114
+ /**
115
+ * Check an optional list parameter (`fq`, `facet_field`): each entry with `itemProblem`, a blank
116
+ * entry refused, never dropped. An empty list means "not given", as before.
117
+ */
118
+ function assertList(name, value, itemProblem) {
119
+ if (value === undefined || (Array.isArray(value) && value.length === 0))
120
+ return;
121
+ assertValid(name, value, (v) => listProblem(v, itemProblem));
93
122
  }
94
123
  /** A JSON object (not null, not an array). */
95
124
  function isObject(value) {
@@ -102,9 +131,9 @@ function isSearchResult(value) {
102
131
  const count = value["count"];
103
132
  return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 && Array.isArray(value["results"]);
104
133
  }
105
- /** The error for a `result` that does not have the shape the caller relies on. */
106
- function shapeError(name, expected) {
107
- return new CkanParseError(`Unexpected response shape from ${ACTION}/${name}: expected ${expected}.`);
134
+ /** The error for a `result` that does not have the shape the caller relies on, naming the request (`GET <url>`). */
135
+ function shapeError(request, expected) {
136
+ return new CkanParseError(`${request}: unexpected response shape: expected ${expected}.`);
108
137
  }
109
138
  /**
110
139
  * Check a `*_list` limit. CKAN reads `limit=0` as "no limit" (a whole catalogue,
@@ -146,43 +175,61 @@ export class CkanClient {
146
175
  #sleep;
147
176
  constructor(options = {}) {
148
177
  this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
149
- // The engine checks every option, `sleep` included, before the client uses it.
150
- this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
178
+ // The engine checks every option, `sleep` included, before the client uses it. Solr's
179
+ // `_version_` (≈1.88e18, past 2^53) is always declared a big integer: a portal that sends it
180
+ // as a number gets the string Hamburg and GovData send, never a refused answer.
181
+ this.#engine = new RequestEngine({
182
+ ...options,
183
+ baseUrl: this.#site,
184
+ bigIntegerKeys: new Set([...bigIntegerKeysOption(options.bigIntegerKeys), ...BIG_INTEGER_KEYS]),
185
+ });
151
186
  this.#sleep = options.sleep ?? realSleep;
152
187
  }
153
188
  /** Call any CKAN action by name and return its unwrapped `result`. */
154
189
  async action(name, params = {}) {
190
+ return (await this.call(name, params)).result;
191
+ }
192
+ /**
193
+ * Call an action: its unwrapped `result` and the request as error messages name it
194
+ * (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
195
+ * with that request.
196
+ */
197
+ async call(name, params) {
155
198
  if (typeof name !== "string" || !ACTION_NAME.test(name)) {
156
- throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
199
+ throw new CkanValidationError(`Invalid CKAN action name: ${quoteValue(typeof name === "string" ? name : String(name))}`);
157
200
  }
158
- const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
201
+ const { value: env, url, request } = await this.#engine.getAnswer(`${ACTION}/${name}`, prune(params));
159
202
  // Another JSON API at the same path (or a proxy's JSON error page) is not an
160
203
  // envelope; say so rather than "failed: undefined".
161
204
  if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
162
- throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
205
+ throw new CkanParseError(`${request}: the answer is not a CKAN Action API response; is ${cutForMessage(redactUrl(this.#site))} a CKAN site?`);
163
206
  }
164
207
  if (!env.success) {
165
208
  // A success:false envelope can come with HTTP 200, so it never passes the
166
209
  // engine's error-detail sanitising: strip terminal controls here too.
167
210
  const type = env.error?.__type;
168
- throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
211
+ throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined, url });
169
212
  }
170
213
  // `{"success": true}` without a result would print nothing useful (and the CLI
171
214
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
172
215
  if (env.result === undefined)
173
- throw shapeError(name, "a result in the envelope");
174
- return env.result;
216
+ throw shapeError(request, "a result in the envelope");
217
+ return { result: env.result, request };
175
218
  }
176
219
  /**
177
220
  * An action whose `result` must have a known top-level shape (never a deep
178
221
  * schema): a broken or foreign answer becomes a CkanParseError naming the
179
- * action, not a TypeError further down (`page is not iterable`).
222
+ * request, not a TypeError further down (`page is not iterable`).
180
223
  */
181
224
  async typed(name, params, ok, expected) {
182
- const result = await this.action(name, params);
225
+ return (await this.typedCall(name, params, ok, expected)).result;
226
+ }
227
+ /** `typed`, with the request it made (for the `all_fields` pager's message). */
228
+ async typedCall(name, params, ok, expected) {
229
+ const { result, request } = await this.call(name, params);
183
230
  if (!ok(result))
184
- throw shapeError(name, expected);
185
- return result;
231
+ throw shapeError(request, expected);
232
+ return { result: result, request };
186
233
  }
187
234
  /** Site title, CKAN version and enabled extensions of the portal. */
188
235
  status() {
@@ -197,30 +244,31 @@ export class CkanClient {
197
244
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
198
245
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
199
246
  * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
200
- * refused (CkanValidationError), never sent or dropped.
247
+ * refused (CkanValidationError), never sent or dropped. The search text `q` and each
248
+ * filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
249
+ * expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
250
+ * a Solr field name) as tokens (`normalizeToken`).
201
251
  */
202
252
  async packageSearch(params = {}) {
203
253
  assertParams(params);
204
254
  assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
205
- assertText("q", params.q);
206
- if (params.fq !== undefined)
207
- assertValid("fq", params.fq, textListProblem);
208
- assertText("sort", params.sort);
209
- if (params.facet_field !== undefined)
210
- assertValid("facet_field", params.facet_field, textListProblem);
255
+ assertOptional("q", params.q, textProblem);
256
+ assertList("fq", params.fq, textProblem);
257
+ assertOptional("sort", params.sort, sortProblem);
258
+ assertList("facet_field", params.facet_field, fieldProblem);
211
259
  assertCount("rows", params.rows);
212
260
  assertCount("start", params.start);
213
261
  if (params.facet_limit !== undefined)
214
262
  assertValid("facet_limit", params.facet_limit, facetLimitProblem);
215
- const fq = params.fq ?? [];
216
- const facetFields = params.facet_field ?? [];
263
+ const fq = (params.fq ?? []).map(normalizeText);
264
+ const facetFields = (params.facet_field ?? []).map(normalizeToken);
217
265
  return this.typed("package_search", {
218
- q: params.q,
266
+ q: params.q === undefined ? undefined : normalizeText(params.q),
219
267
  fq: fq.length === 1 ? fq[0] : undefined,
220
268
  fq_list: fq.length > 1 ? fq : undefined,
221
269
  rows: params.rows,
222
270
  start: params.start,
223
- sort: params.sort,
271
+ sort: params.sort === undefined ? undefined : normalizeToken(params.sort),
224
272
  "facet.field": facetFields.length > 0 ? JSON.stringify(facetFields) : undefined,
225
273
  "facet.limit": params.facet_limit,
226
274
  }, isSearchResult, "an object with a numeric count and a results array");
@@ -260,12 +308,13 @@ export class CkanClient {
260
308
  groupList(params = {}) {
261
309
  return this.groupOrOrgList("group_list", params);
262
310
  }
263
- /** Tags, optionally only those containing a substring (a blank one is refused). */
311
+ /** Tags, optionally only those containing a substring (a blank one is refused; sent trimmed, in NFC). */
264
312
  async tagList(params = {}) {
265
313
  assertParams(params);
266
314
  assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
267
- assertText("query", params.query);
268
- return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
315
+ assertOptional("query", params.query, textProblem);
316
+ const query = params.query === undefined ? undefined : normalizeText(params.query);
317
+ return this.typed("tag_list", { query }, Array.isArray, "an array");
269
318
  }
270
319
  /** The licences this portal offers. */
271
320
  licenseList() {
@@ -312,20 +361,24 @@ export class CkanClient {
312
361
  // Set after a short window came back empty (or with nothing new): from then on
313
362
  // every page is a full one.
314
363
  let fullPages = false;
364
+ /** The last page's request, which the stop message names. */
365
+ let request = "";
315
366
  while (out.length < wanted) {
316
367
  if (pages >= MAX_ALL_FIELDS_PAGES) {
317
- throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
368
+ throw new CkanParseError(`${request}: ${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
318
369
  `reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
319
370
  `Pass a limit (--limit) to fetch part of it.`);
320
371
  }
321
372
  if (pages > 0)
322
373
  await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
323
374
  const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
324
- const page = await this.typed(action, {
375
+ const answer = await this.typedCall(action, {
325
376
  all_fields: true,
326
377
  limit,
327
378
  offset: offset === 0 ? undefined : offset,
328
379
  }, Array.isArray, "an array");
380
+ const page = answer.result;
381
+ request = answer.request;
329
382
  pages += 1;
330
383
  let added = 0;
331
384
  for (const entry of page) {
@@ -350,12 +403,12 @@ export class CkanClient {
350
403
  return out.slice(0, wanted);
351
404
  }
352
405
  /**
353
- * A `*_show` call. A blank id is rejected up front: `prune` would drop it and
354
- * CKAN would answer with a 409 validation error instead.
406
+ * A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
407
+ * invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
408
+ * blank one and CKAN would answer with a 409 validation error instead.
355
409
  */
356
410
  async show(action, id) {
357
- if (typeof id !== "string" || id.trim() === "")
358
- throw new CkanValidationError(`${action} needs an id or name.`);
359
- return this.typed(action, { id }, isObject, "a JSON object");
411
+ assertValid("id", id, idProblem);
412
+ return this.typed(action, { id: normalizeToken(id) }, isObject, "a JSON object");
360
413
  }
361
414
  }
@@ -1,5 +1,6 @@
1
1
  import { type Transport } from "./http.js";
2
2
  import { type QueryParams } from "./query.js";
3
+ export { quoteValue } from "./errors.js";
3
4
  export declare const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
4
5
  export interface RawResponse {
5
6
  data: Buffer;
@@ -10,35 +11,35 @@ export interface RawResponse {
10
11
  }
11
12
  export interface EngineOptions {
12
13
  /** Base URL of the API. Defaults to https://suche.transparenz.hamburg.de (the Hamburg Transparenzportal) */
13
- baseUrl?: string;
14
+ baseUrl?: string | undefined;
14
15
  /** Swappable transport. Defaults to the built-in node http/https transport. */
15
- transport?: Transport;
16
+ transport?: Transport | undefined;
16
17
  /**
17
18
  * Value of the User-Agent header. Must be sendable (see `assertHeaderValue`):
18
19
  * not blank, no control characters but tab, nothing above U+00FF.
19
20
  */
20
- userAgent?: string;
21
+ userAgent?: string | undefined;
21
22
  /**
22
23
  * Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
23
24
  * It covers the whole response, body included, and the engine enforces it for every
24
25
  * transport (the request also gets an AbortSignal that fires at the deadline).
25
26
  */
26
- timeoutMs?: number;
27
+ timeoutMs?: number | undefined;
27
28
  /**
28
29
  * Number of automatic retries for transient (429/503) responses; a reset or other
29
30
  * network error is not retried. Each waits `retryDelayMs * attempt`, or the
30
31
  * response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
31
32
  * one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
32
33
  */
33
- maxRetries?: number;
34
+ maxRetries?: number | undefined;
34
35
  /**
35
36
  * Base backoff between retries in milliseconds (grows linearly), 0 to
36
37
  * `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
37
38
  * `Retry-After`: the header can lengthen a wait, never shorten it.
38
39
  */
39
- retryDelayMs?: number;
40
+ retryDelayMs?: number | undefined;
40
41
  /** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
41
- maxRedirects?: number;
42
+ maxRedirects?: number | undefined;
42
43
  /**
43
44
  * Hard cap on response body size in bytes (defends against memory exhaustion
44
45
  * from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
@@ -49,14 +50,34 @@ export interface EngineOptions {
49
50
  * the constructor throws CkanValidationError otherwise (a negative or NaN
50
51
  * timeout or cap would silently switch that guard off).
51
52
  */
52
- maxResponseBytes?: number;
53
+ maxResponseBytes?: number | undefined;
53
54
  /** Injectable sleep, primarily for deterministic tests. */
54
- sleep?: (ms: number) => Promise<void>;
55
+ sleep?: ((ms: number) => Promise<void>) | undefined;
55
56
  /**
56
57
  * Called once per retry, right before the backoff sleep, for each retried 429/503
57
58
  * and reset connection; never when there is no retry. A throw is swallowed.
58
59
  */
59
- onRetry?: (event: RetryEvent) => void;
60
+ onRetry?: ((event: RetryEvent) => void) | undefined;
61
+ /**
62
+ * Called with a one-line warning about an answer the engine delivered changed: characters
63
+ * that were not valid text — invalid bytes in the declared (or default UTF-8) charset, or a
64
+ * lone surrogate written as a JSON escape — replaced with U+FFFD. The message names the
65
+ * request and the count (`GET <url>: 2 characters of the answer were not valid text …`).
66
+ * Without it the replacement is silent. A throw is swallowed. The CLI logs it as a `WARN`
67
+ * record of `ckan.api`.
68
+ */
69
+ onWarning?: ((message: string) => void) | undefined;
70
+ /**
71
+ * Member names whose number values may exceed 2^53 (a Solr `_version_`, a 64-bit id). The value
72
+ * of a declared member is delivered as its exact literal, as written, as a JSON string (`"5"`),
73
+ * so its type never depends on its size. A declared member whose value is not a number (a
74
+ * string, `null`, an object) is left as it is. Every undeclared number keeps the rule: exact,
75
+ * or a CkanParseError. Matched by name anywhere in a JSON answer, an escaped name
76
+ * (`"\u005fversion_"`) included. Defaults to none in the engine (the CkanClient always adds
77
+ * `_version_`); anything but a set or list of non-empty strings makes the constructor throw a
78
+ * CkanValidationError. The set is copied.
79
+ */
80
+ bigIntegerKeys?: ReadonlySet<string> | undefined;
60
81
  }
61
82
  /** What `EngineOptions.onRetry` is told about one retry. */
62
83
  export interface RetryEvent {
@@ -147,6 +168,11 @@ export declare function validateBaseUrl(raw: string): string;
147
168
  * (once per run, before the first request).
148
169
  */
149
170
  export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
171
+ /**
172
+ * Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
173
+ * copied; anything else throws an CkanValidationError.
174
+ */
175
+ export declare function bigIntegerKeysOption(value: ReadonlySet<string> | undefined): ReadonlySet<string>;
150
176
  export declare class RequestEngine {
151
177
  #private;
152
178
  private readonly transport;
@@ -158,7 +184,12 @@ export declare class RequestEngine {
158
184
  private readonly maxResponseBytes;
159
185
  private readonly sleep;
160
186
  private readonly onRetry;
187
+ private readonly onWarning;
188
+ /** `EngineOptions.bigIntegerKeys`, copied. */
189
+ readonly bigIntegerKeys: ReadonlySet<string>;
161
190
  constructor(options?: EngineOptions);
191
+ /** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
192
+ private warn;
162
193
  /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
163
194
  private backOff;
164
195
  /**
@@ -197,7 +228,55 @@ export declare class RequestEngine {
197
228
  query?: QueryParams;
198
229
  accept: string;
199
230
  }): Promise<RawResponse>;
200
- /** Perform a GET expecting JSON and parse it into `T`. */
231
+ /** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
201
232
  getJson<T>(path: string, query?: QueryParams): Promise<T>;
233
+ /**
234
+ * Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
235
+ * userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
236
+ * MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
237
+ * (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
238
+ * one failed; the client prefixes its own envelope and shape errors with it too.
239
+ *
240
+ * The text is never changed silently: bytes that are not valid in the charset, and lone
241
+ * surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
242
+ * well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
243
+ * `onWarning` is told how many. One bad character never sinks a whole answer.
244
+ */
245
+ getAnswer<T>(path: string, query?: QueryParams): Promise<{
246
+ value: T;
247
+ url: string;
248
+ request: string;
249
+ }>;
202
250
  private toApiError;
203
251
  }
252
+ /** What {@link scanNumbers} found in a JSON text. */
253
+ interface NumberScan {
254
+ /** The text with each declared member's number value quoted; the input itself when there was none. */
255
+ json: string;
256
+ /** The first undeclared number literal JSON.parse cannot carry exactly, or undefined. */
257
+ inexact: string | undefined;
258
+ }
259
+ /**
260
+ * One linear pass over `json` (the text of an answer) that skips strings and looks at every
261
+ * number literal:
262
+ * - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
263
+ * 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
264
+ * it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
265
+ * so a malformed one (`01`) still fails JSON.parse;
266
+ * - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
267
+ * cannot carry exactly is reported in `inexact`.
268
+ * Nothing is allocated per value: a member name is compared in place (only a name written with
269
+ * an escape is decoded), and the text is copied only when a declared member is found. No
270
+ * reviver, which would recurse per nesting level and overflow on deep data.
271
+ */
272
+ export declare function scanNumbers(json: string, bigIntegerKeys?: ReadonlySet<string>): NumberScan;
273
+ /**
274
+ * The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
275
+ * undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
276
+ * 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
277
+ * `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
278
+ * (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
279
+ * `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
280
+ * precision, as JSON means it. {@link scanNumbers} without declared members.
281
+ */
282
+ export declare function firstInexactNumber(json: string): string | undefined;