@maschinenlesbar.org/ckan-cli 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. package/README.md +21 -7
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +16 -5
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +98 -47
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +106 -5
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +435 -80
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.d.ts +68 -7
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/errors.js +188 -18
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.d.ts +2 -1
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/http.js +33 -6
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -2
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -2
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +4 -1
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +11 -3
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +72 -13
  15. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +190 -22
  16. package/dist/src/cli/commands/catalogue.js +12 -11
  17. package/dist/src/cli/commands/portal.js +28 -20
  18. package/dist/src/cli/io.d.ts +16 -0
  19. package/dist/src/cli/io.js +55 -1
  20. package/dist/src/cli/output.d.ts +26 -0
  21. package/dist/src/cli/output.js +151 -0
  22. package/dist/src/cli/run.d.ts +0 -7
  23. package/dist/src/cli/run.js +80 -17
  24. package/dist/src/cli/shared.d.ts +14 -25
  25. package/dist/src/cli/shared.js +40 -102
  26. package/package.json +1 -1
package/README.md CHANGED
@@ -155,8 +155,14 @@ action <name> [--param key=value …] call any CKAN action (generic)
155
155
  > **Several `--fq` filters** are sent as CKAN's `fq_list`, never as a repeated
156
156
  > `fq` key, which CKAN answers with HTTP 409.
157
157
 
158
- > **Blank values are refused.** `ckan search ""`, `--fq ""`, `tags --query ""` or
159
- > `action … --param q=` exit 1 before any request, instead of silently running an unfiltered search.
158
+ > **Values go out canonical, or are refused.** The search text, `--fq`, `tags --query`
159
+ > and `--param` values are sent trimmed and in composed Unicode (NFC); ids, `--sort`,
160
+ > `--facet` and `--param` names trimmed. A blank value (`ckan search ""`, `--fq ""`,
161
+ > `tags --query ""`, `action … --param q=`, or one of only invisible characters) exits 1
162
+ > before any request, instead of silently running an unfiltered search; so does a value
163
+ > with a control or invisible character (a zero-width space pasted from a web page), shown
164
+ > escaped (`"el\u200bbe"`), a `--sort` that is not `<field> asc|desc[, …]`, a `--facet`
165
+ > that is not a field name, and an id with a `/`.
160
166
  > So does a forgotten value (`--fq --rows 5`): a value of `--fq`, `--facet`,
161
167
  > `--sort` or `tags --query` cannot start with `--`. And so does a single-value
162
168
  > option given twice (`--rows 5 --rows 50`, a second `--base-url`): only `--fq`,
@@ -222,8 +228,10 @@ Found by running the same commands against many portals:
222
228
  - **Solr syntax errors** come back as HTTP 409; `ckan` prints Solr's own reason,
223
229
  e.g. `Search Error: Cannot parse 'title:(': Encountered "<EOF>" …`.
224
230
  - **A site that is not (or no longer) a CKAN** answers with a 404, an HTML page or
225
- a redirect loop; `ckan` says which, e.g.
226
- `Expected JSON from … but got text/html`.
231
+ a redirect loop; `ckan` says which, naming the request, e.g.
232
+ `GET <url>: the answer is not valid JSON (expected JSON, got Content-Type "text/html").`
233
+ Every error about an answer starts with the request (`GET <url>: …`), CKAN's own error
234
+ answer sent with HTTP 200 too, so a run over many portals tells which one failed.
227
235
 
228
236
  ## Output & scripting
229
237
 
@@ -234,7 +242,11 @@ prints one line. Every global option works before or after the command.
234
242
  Each line on stderr is a **log record**: a timestamp (UTC), a level (`ERROR`, `WARN`,
235
243
  `INFO`) and a topic, the program and the area it comes from (`ckan.cli` for usage
236
244
  errors, `ckan.api` for the portal's answers, a malformed one included, `ckan.http` for the connection,
237
- `ckan.output` when stdout can't be written). By default
245
+ `ckan.output` when stdout can't be written, a redirected file cut short by a full disk
246
+ included, or an answer can't be printed). Text in an
247
+ answer is never changed silently: bytes that are not valid in its charset and half
248
+ characters (lone surrogates) are replaced with U+FFFD, so the output stays readable for
249
+ `jq`, and a `WARN` record of `ckan.api` says how many. By default
238
250
  it is written log4j style; `--log-format jsonl` writes one JSON object per line instead.
239
251
  A record is always one line: a line break, a control character or a bidi control in a
240
252
  message (a server's text, a value you typed) is written as an escape (`\n`, `\u001b`,
@@ -279,9 +291,11 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
279
291
  > **Numbers pass through JavaScript.** The output is the parsed and re-serialised
280
292
  > JSON. A number that would not print as the server wrote it (an integer above 2⁵³,
281
293
  > 9,007,199,254,740,992, that loses its last digits, a number beyond the double range,
282
- > `1e400`, or one too small, `1e-400`) is an error (exit 1, `the number … cannot be
294
+ > `1e400`, one too small, `1e-400`, or one below the normal double range that a double
295
+ > would change, `3e-324`) is an error (exit 1, `the number … cannot be
283
296
  > represented exactly`), never a changed value; one the server wrote in a double's
284
- > shortest form (`1E23`) prints as written. CKAN metadata rarely holds such numbers; for
297
+ > shortest form (`1E23`) prints as written. Solr's `_version_` (`--param fl=_version_`) is
298
+ > always printed as a string with its exact digits, as Hamburg and GovData send it. CKAN metadata rarely holds such numbers; for
285
299
  > an exact copy of the raw answer, use `curl` on `<site>/api/3/action/<name>`.
286
300
 
287
301
  ## Global options
@@ -27,12 +27,20 @@ export declare class CkanClient {
27
27
  constructor(options?: EngineOptions);
28
28
  /** Call any CKAN action by name and return its unwrapped `result`. */
29
29
  action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
30
+ /**
31
+ * Call an action: its unwrapped `result` and the request as error messages name it
32
+ * (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
33
+ * with that request.
34
+ */
35
+ private call;
30
36
  /**
31
37
  * An action whose `result` must have a known top-level shape (never a deep
32
38
  * schema): a broken or foreign answer becomes a CkanParseError naming the
33
- * action, not a TypeError further down (`page is not iterable`).
39
+ * request, not a TypeError further down (`page is not iterable`).
34
40
  */
35
41
  private typed;
42
+ /** `typed`, with the request it made (for the `all_fields` pager's message). */
43
+ private typedCall;
36
44
  /** Site title, CKAN version and enabled extensions of the portal. */
37
45
  status(): Promise<Status>;
38
46
  /**
@@ -44,8 +52,10 @@ export declare class CkanClient {
44
52
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
45
53
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
46
54
  * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
47
- * refused (CkanValidationError), never sent or dropped. The search text `q` goes
48
- * out trimmed and in NFC (`normalizeText`); filters, sort and facet fields as given.
55
+ * refused (CkanValidationError), never sent or dropped. The search text `q` and each
56
+ * filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
57
+ * expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
58
+ * a Solr field name) as tokens (`normalizeToken`).
49
59
  */
50
60
  packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
51
61
  /** A single dataset by id or name. */
@@ -92,8 +102,9 @@ export declare class CkanClient {
92
102
  */
93
103
  private groupOrOrgList;
94
104
  /**
95
- * A `*_show` call. A blank id is rejected up front: `prune` would drop it and
96
- * CKAN would answer with a 409 validation error instead.
105
+ * A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
106
+ * invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
107
+ * blank one and CKAN would answer with a 409 validation error instead.
97
108
  */
98
109
  private show;
99
110
  }
@@ -1,9 +1,15 @@
1
1
  // CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
2
2
  // Action API (`<site>/api/3/action`).
3
- import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
- import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
5
- import { assertValid, blankProblem, countProblem, facetLimitProblem, normalizeText, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
3
+ import { bigIntegerKeysOption, DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
+ import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, quoteValue, redactUrl } from "./errors.js";
5
+ import { assertValid, countProblem, facetLimitProblem, fieldProblem, idProblem, listProblem, normalizeText, normalizeToken, queryValueProblem, sortProblem, textProblem, tokenProblem, } from "./validate.js";
6
6
  const ACTION = "/api/3/action";
7
+ /**
8
+ * The members every CkanClient declares as big integers (`EngineOptions.bigIntegerKeys`): Solr's
9
+ * `_version_`, which `package_search` returns for `fl=_version_` (Hamburg and GovData send it as
10
+ * a string; a number would be past 2^53).
11
+ */
12
+ const BIG_INTEGER_KEYS = ["_version_"];
7
13
  /**
8
14
  * CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
9
15
  * the path-traversal hole (`../../..` escaping `/api/3/action/`) and the
@@ -37,26 +43,40 @@ function entryKey(entry) {
37
43
  return JSON.stringify(entry);
38
44
  }
39
45
  /**
40
- * Drop undefined values so only the parameters the caller actually set are sent,
41
- * and refuse a blank parameter name, a blank value or a list with a blank entry
42
- * (CkanValidationError). CKAN reads an empty parameter as "not given", so a blank
43
- * filter such as `tagList({ query: "" })` would silently return everything.
46
+ * Drop undefined values so only the parameters the caller actually set are sent, and send
47
+ * each name as a token (`normalizeToken`: trimmed, NFC) and each string value as text
48
+ * (`normalizeText`). A blank or invisible name, a blank value, a value with a control or
49
+ * format character, and two names that are the same once trimmed (`q` and ` q`) are refused
50
+ * (CkanValidationError): CKAN reads an empty parameter as "not given", so a blank filter such
51
+ * as `tagList({ query: "" })` would silently return everything.
44
52
  */
45
53
  function prune(params) {
46
54
  assertParams(params);
47
55
  // A null-prototype object, so a `__proto__` key is kept as a parameter instead
48
56
  // of setting the prototype (and being lost).
49
57
  const out = Object.create(null);
58
+ const given = new Map();
50
59
  for (const [k, v] of Object.entries(params)) {
51
60
  if (v === undefined)
52
61
  continue;
53
- assertValid("parameter name", k, blankProblem);
62
+ const name = normalizeToken(assertValid("parameter name", k, tokenProblem));
63
+ const earlier = given.get(name);
64
+ if (earlier !== undefined) {
65
+ throw new CkanValidationError(`Invalid params: ${quoteValue(name)} is given twice (as ${quoteValue(earlier)} and ${quoteValue(k)}).`);
66
+ }
67
+ given.set(name, k);
54
68
  // A value the query string can't carry (an object, NaN, a function) is refused,
55
69
  // not sent as "[object Object]" or "NaN".
56
- out[k] = assertValid(k, v, queryValueProblem);
70
+ out[name] = textValue(assertValid(k, v, queryValueProblem));
57
71
  }
58
72
  return out;
59
73
  }
74
+ /** A query value with each string in it sent as text (`normalizeText`); other values as they are. */
75
+ function textValue(value) {
76
+ if (typeof value === "string")
77
+ return normalizeText(value);
78
+ return Array.isArray(value) ? value.map((v) => (typeof v === "string" ? normalizeText(v) : v)) : value;
79
+ }
60
80
  /**
61
81
  * A method's parameter object: a plain object (or undefined, for the defaults).
62
82
  * `null`, a string or an array would otherwise fail as a raw TypeError.
@@ -77,7 +97,7 @@ function assertParams(params) {
77
97
  function assertKeys(method, action, params, known) {
78
98
  for (const key of Object.keys(params)) {
79
99
  if (!known.includes(key)) {
80
- throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
100
+ throw new CkanValidationError(`Invalid ${method} parameter ${quoteValue(key)}: not a parameter of ${method}. ` +
81
101
  `Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
82
102
  }
83
103
  }
@@ -86,10 +106,19 @@ const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_l
86
106
  const PACKAGE_LIST_KEYS = ["limit", "offset"];
87
107
  const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
88
108
  const TAG_LIST_KEYS = ["query"];
89
- /** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
90
- function assertText(name, value) {
109
+ /** Check an optional parameter with `problem`; `undefined` means "not given". */
110
+ function assertOptional(name, value, problem) {
91
111
  if (value !== undefined)
92
- assertValid(name, value, textProblem);
112
+ assertValid(name, value, problem);
113
+ }
114
+ /**
115
+ * Check an optional list parameter (`fq`, `facet_field`): each entry with `itemProblem`, a blank
116
+ * entry refused, never dropped. An empty list means "not given", as before.
117
+ */
118
+ function assertList(name, value, itemProblem) {
119
+ if (value === undefined || (Array.isArray(value) && value.length === 0))
120
+ return;
121
+ assertValid(name, value, (v) => listProblem(v, itemProblem));
93
122
  }
94
123
  /** A JSON object (not null, not an array). */
95
124
  function isObject(value) {
@@ -102,9 +131,9 @@ function isSearchResult(value) {
102
131
  const count = value["count"];
103
132
  return typeof count === "number" && Number.isSafeInteger(count) && count >= 0 && Array.isArray(value["results"]);
104
133
  }
105
- /** The error for a `result` that does not have the shape the caller relies on. */
106
- function shapeError(name, expected) {
107
- return new CkanParseError(`Unexpected response shape from ${ACTION}/${name}: expected ${expected}.`);
134
+ /** The error for a `result` that does not have the shape the caller relies on, naming the request (`GET <url>`). */
135
+ function shapeError(request, expected) {
136
+ return new CkanParseError(`${request}: unexpected response shape: expected ${expected}.`);
108
137
  }
109
138
  /**
110
139
  * Check a `*_list` limit. CKAN reads `limit=0` as "no limit" (a whole catalogue,
@@ -146,43 +175,61 @@ export class CkanClient {
146
175
  #sleep;
147
176
  constructor(options = {}) {
148
177
  this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
149
- // The engine checks every option, `sleep` included, before the client uses it.
150
- this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
178
+ // The engine checks every option, `sleep` included, before the client uses it. Solr's
179
+ // `_version_` (≈1.88e18, past 2^53) is always declared a big integer: a portal that sends it
180
+ // as a number gets the string Hamburg and GovData send, never a refused answer.
181
+ this.#engine = new RequestEngine({
182
+ ...options,
183
+ baseUrl: this.#site,
184
+ bigIntegerKeys: new Set([...bigIntegerKeysOption(options.bigIntegerKeys), ...BIG_INTEGER_KEYS]),
185
+ });
151
186
  this.#sleep = options.sleep ?? realSleep;
152
187
  }
153
188
  /** Call any CKAN action by name and return its unwrapped `result`. */
154
189
  async action(name, params = {}) {
190
+ return (await this.call(name, params)).result;
191
+ }
192
+ /**
193
+ * Call an action: its unwrapped `result` and the request as error messages name it
194
+ * (`GET <url>`, the URL that answered, userinfo redacted). Every error about the answer starts
195
+ * with that request.
196
+ */
197
+ async call(name, params) {
155
198
  if (typeof name !== "string" || !ACTION_NAME.test(name)) {
156
- throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
199
+ throw new CkanValidationError(`Invalid CKAN action name: ${quoteValue(typeof name === "string" ? name : String(name))}`);
157
200
  }
158
- const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
201
+ const { value: env, url, request } = await this.#engine.getAnswer(`${ACTION}/${name}`, prune(params));
159
202
  // Another JSON API at the same path (or a proxy's JSON error page) is not an
160
203
  // envelope; say so rather than "failed: undefined".
161
204
  if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
162
- throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
205
+ throw new CkanParseError(`${request}: the answer is not a CKAN Action API response; is ${cutForMessage(redactUrl(this.#site))} a CKAN site?`);
163
206
  }
164
207
  if (!env.success) {
165
208
  // A success:false envelope can come with HTTP 200, so it never passes the
166
209
  // engine's error-detail sanitising: strip terminal controls here too.
167
210
  const type = env.error?.__type;
168
- throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
211
+ throw new CkanActionError(`${request}: CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(this.#engine.scrub(type))) : undefined, url });
169
212
  }
170
213
  // `{"success": true}` without a result would print nothing useful (and the CLI
171
214
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
172
215
  if (env.result === undefined)
173
- throw shapeError(name, "a result in the envelope");
174
- return env.result;
216
+ throw shapeError(request, "a result in the envelope");
217
+ return { result: env.result, request };
175
218
  }
176
219
  /**
177
220
  * An action whose `result` must have a known top-level shape (never a deep
178
221
  * schema): a broken or foreign answer becomes a CkanParseError naming the
179
- * action, not a TypeError further down (`page is not iterable`).
222
+ * request, not a TypeError further down (`page is not iterable`).
180
223
  */
181
224
  async typed(name, params, ok, expected) {
182
- const result = await this.action(name, params);
225
+ return (await this.typedCall(name, params, ok, expected)).result;
226
+ }
227
+ /** `typed`, with the request it made (for the `all_fields` pager's message). */
228
+ async typedCall(name, params, ok, expected) {
229
+ const { result, request } = await this.call(name, params);
183
230
  if (!ok(result))
184
- throw shapeError(name, expected);
185
- return result;
231
+ throw shapeError(request, expected);
232
+ return { result: result, request };
186
233
  }
187
234
  /** Site title, CKAN version and enabled extensions of the portal. */
188
235
  status() {
@@ -197,31 +244,31 @@ export class CkanClient {
197
244
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
198
245
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
199
246
  * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
200
- * refused (CkanValidationError), never sent or dropped. The search text `q` goes
201
- * out trimmed and in NFC (`normalizeText`); filters, sort and facet fields as given.
247
+ * refused (CkanValidationError), never sent or dropped. The search text `q` and each
248
+ * filter go out as text, trimmed and in NFC (`normalizeText`, `textProblem`); the sort
249
+ * expression (`sortProblem`: `<field> asc|desc`, …) and each facet field (`fieldProblem`:
250
+ * a Solr field name) as tokens (`normalizeToken`).
202
251
  */
203
252
  async packageSearch(params = {}) {
204
253
  assertParams(params);
205
254
  assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
206
- assertText("q", params.q);
207
- if (params.fq !== undefined)
208
- assertValid("fq", params.fq, textListProblem);
209
- assertText("sort", params.sort);
210
- if (params.facet_field !== undefined)
211
- assertValid("facet_field", params.facet_field, textListProblem);
255
+ assertOptional("q", params.q, textProblem);
256
+ assertList("fq", params.fq, textProblem);
257
+ assertOptional("sort", params.sort, sortProblem);
258
+ assertList("facet_field", params.facet_field, fieldProblem);
212
259
  assertCount("rows", params.rows);
213
260
  assertCount("start", params.start);
214
261
  if (params.facet_limit !== undefined)
215
262
  assertValid("facet_limit", params.facet_limit, facetLimitProblem);
216
- const fq = params.fq ?? [];
217
- const facetFields = params.facet_field ?? [];
263
+ const fq = (params.fq ?? []).map(normalizeText);
264
+ const facetFields = (params.facet_field ?? []).map(normalizeToken);
218
265
  return this.typed("package_search", {
219
266
  q: params.q === undefined ? undefined : normalizeText(params.q),
220
267
  fq: fq.length === 1 ? fq[0] : undefined,
221
268
  fq_list: fq.length > 1 ? fq : undefined,
222
269
  rows: params.rows,
223
270
  start: params.start,
224
- sort: params.sort,
271
+ sort: params.sort === undefined ? undefined : normalizeToken(params.sort),
225
272
  "facet.field": facetFields.length > 0 ? JSON.stringify(facetFields) : undefined,
226
273
  "facet.limit": params.facet_limit,
227
274
  }, isSearchResult, "an object with a numeric count and a results array");
@@ -265,7 +312,7 @@ export class CkanClient {
265
312
  async tagList(params = {}) {
266
313
  assertParams(params);
267
314
  assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
268
- assertText("query", params.query);
315
+ assertOptional("query", params.query, textProblem);
269
316
  const query = params.query === undefined ? undefined : normalizeText(params.query);
270
317
  return this.typed("tag_list", { query }, Array.isArray, "an array");
271
318
  }
@@ -314,20 +361,24 @@ export class CkanClient {
314
361
  // Set after a short window came back empty (or with nothing new): from then on
315
362
  // every page is a full one.
316
363
  let fullPages = false;
364
+ /** The last page's request, which the stop message names. */
365
+ let request = "";
317
366
  while (out.length < wanted) {
318
367
  if (pages >= MAX_ALL_FIELDS_PAGES) {
319
- throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
368
+ throw new CkanParseError(`${request}: ${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
320
369
  `reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
321
370
  `Pass a limit (--limit) to fetch part of it.`);
322
371
  }
323
372
  if (pages > 0)
324
373
  await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
325
374
  const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
326
- const page = await this.typed(action, {
375
+ const answer = await this.typedCall(action, {
327
376
  all_fields: true,
328
377
  limit,
329
378
  offset: offset === 0 ? undefined : offset,
330
379
  }, Array.isArray, "an array");
380
+ const page = answer.result;
381
+ request = answer.request;
331
382
  pages += 1;
332
383
  let added = 0;
333
384
  for (const entry of page) {
@@ -352,12 +403,12 @@ export class CkanClient {
352
403
  return out.slice(0, wanted);
353
404
  }
354
405
  /**
355
- * A `*_show` call. A blank id is rejected up front: `prune` would drop it and
356
- * CKAN would answer with a 409 validation error instead.
406
+ * A `*_show` call. The id goes out as a token (`normalizeToken`: trimmed, NFC); a blank or
407
+ * invisible one, or one with a `/`, is refused up front (`idProblem`): `prune` would drop a
408
+ * blank one and CKAN would answer with a 409 validation error instead.
357
409
  */
358
410
  async show(action, id) {
359
- if (typeof id !== "string" || id.trim() === "")
360
- throw new CkanValidationError(`${action} needs an id or name.`);
361
- return this.typed(action, { id }, isObject, "a JSON object");
411
+ assertValid("id", id, idProblem);
412
+ return this.typed(action, { id: normalizeToken(id) }, isObject, "a JSON object");
362
413
  }
363
414
  }
@@ -1,5 +1,6 @@
1
1
  import { type Transport } from "./http.js";
2
2
  import { type QueryParams } from "./query.js";
3
+ export { quoteValue } from "./errors.js";
3
4
  export declare const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
4
5
  export interface RawResponse {
5
6
  data: Buffer;
@@ -57,6 +58,26 @@ export interface EngineOptions {
57
58
  * and reset connection; never when there is no retry. A throw is swallowed.
58
59
  */
59
60
  onRetry?: ((event: RetryEvent) => void) | undefined;
61
+ /**
62
+ * Called with a one-line warning about an answer the engine delivered changed: characters
63
+ * that were not valid text — invalid bytes in the declared (or default UTF-8) charset, or a
64
+ * lone surrogate written as a JSON escape — replaced with U+FFFD. The message names the
65
+ * request and the count (`GET <url>: 2 characters of the answer were not valid text …`).
66
+ * Without it the replacement is silent. A throw is swallowed. The CLI logs it as a `WARN`
67
+ * record of `ckan.api`.
68
+ */
69
+ onWarning?: ((message: string) => void) | undefined;
70
+ /**
71
+ * Member names whose number values may exceed 2^53 (a Solr `_version_`, a 64-bit id). The value
72
+ * of a declared member is delivered as its exact literal, as written, as a JSON string (`"5"`),
73
+ * so its type never depends on its size. A declared member whose value is not a number (a
74
+ * string, `null`, an object) is left as it is. Every undeclared number keeps the rule: exact,
75
+ * or a CkanParseError. Matched by name anywhere in a JSON answer, an escaped name
76
+ * (`"\u005fversion_"`) included. Defaults to none in the engine (the CkanClient always adds
77
+ * `_version_`); anything but a set or list of non-empty strings makes the constructor throw a
78
+ * CkanValidationError. The set is copied.
79
+ */
80
+ bigIntegerKeys?: ReadonlySet<string> | undefined;
60
81
  }
61
82
  /** What `EngineOptions.onRetry` is told about one retry. */
62
83
  export interface RetryEvent {
@@ -110,19 +131,25 @@ export declare function isBidiControl(code: number): boolean;
110
131
  * `detail`, a CKAN `success:false` error, the echoed Content-Type — safe to print
111
132
  * into an error message on stderr:
112
133
  *
113
- * - C0 and C1 controls and DEL are dropped. A JSON error body can encode an escape
134
+ * - Whole escape sequences (CSI, OSC and other control strings) are dropped first,
135
+ * then any remaining C0 and C1 controls and DEL. A JSON error body can encode an escape
114
136
  * (U+001B) that JSON.parse turns into a real control byte; printed raw, a hostile
115
137
  * or MITM'd endpoint could drive ANSI/OSC sequences into the terminal (display
116
138
  * spoofing, title changes, OSC 52 clipboard writes).
117
- * - Bidi formatting characters (isBidiControl) are dropped, so server text cannot
118
- * reorder the visible message.
139
+ * - Unicode format characters (General Category Cf) are dropped: the bidi controls
140
+ * (isBidiControl), so server text cannot reorder the visible message, and the
141
+ * invisible ones — zero-width space/joiners, word joiner, soft hyphen, BOM — so
142
+ * two messages that look identical are identical.
119
143
  * - Every run of whitespace — newlines, tabs, U+2028/U+2029 included — becomes one
120
144
  * space and the ends are trimmed, so the text stays on one line and a server
121
145
  * cannot forge an `Error:` line of its own.
146
+ * - Only the first MAX_SERVER_TEXT (16 384) units are read, and every step is a linear
147
+ * regex replace: a hostile error body costs neither seconds nor the heap. A caller that
148
+ * scrubs credentials does so first, so a cut never splits one.
122
149
  *
123
150
  * The CLI's JSON output is escaped separately (`escapeControlChars` in
124
151
  * cli/shared.ts): `JSON.stringify` alone leaves DEL, C1 and bidi characters raw.
125
- * Written as a char-code filter so no raw control byte appears in this source.
152
+ * Written with escapes so no raw control byte appears in this source.
126
153
  */
127
154
  export declare function sanitizeServerText(text: string): string;
128
155
  /**
@@ -147,6 +174,11 @@ export declare function validateBaseUrl(raw: string): string;
147
174
  * (once per run, before the first request).
148
175
  */
149
176
  export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
177
+ /**
178
+ * Read the `bigIntegerKeys` option: `undefined` gives none; a set or list of non-empty strings is
179
+ * copied; anything else throws an CkanValidationError.
180
+ */
181
+ export declare function bigIntegerKeysOption(value: ReadonlySet<string> | undefined): ReadonlySet<string>;
150
182
  export declare class RequestEngine {
151
183
  #private;
152
184
  private readonly transport;
@@ -158,7 +190,12 @@ export declare class RequestEngine {
158
190
  private readonly maxResponseBytes;
159
191
  private readonly sleep;
160
192
  private readonly onRetry;
193
+ private readonly onWarning;
194
+ /** `EngineOptions.bigIntegerKeys`, copied. */
195
+ readonly bigIntegerKeys: ReadonlySet<string>;
161
196
  constructor(options?: EngineOptions);
197
+ /** Tell `onWarning` about an answer delivered changed. A throwing callback never breaks the request. */
198
+ private warn;
162
199
  /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
163
200
  private backOff;
164
201
  /**
@@ -169,6 +206,15 @@ export declare class RequestEngine {
169
206
  * messages built from server text.
170
207
  */
171
208
  scrub(text: string): string;
209
+ /**
210
+ * An error body, already scrubbed as sent, as `CkanApiError.body` keeps it: when it is JSON
211
+ * with an escape in it, checked decoded too. An escape (`\"`, `\u00e4`) hides an echoed
212
+ * credential from the scrub of the text as sent, while a caller who parses the body sees it. Such
213
+ * a body is kept as the decoded JSON written again (`JSON.stringify`, one spelling per character)
214
+ * with the credential scrubbed in that spelling. A body that doesn't parse is text nobody
215
+ * decodes, and stays as sent; one too deeply nested to write again is left out.
216
+ */
217
+ private scrubBody;
172
218
  /**
173
219
  * A failure as the `cause` of the error the engine raises: the original when its
174
220
  * text carries no credentials, otherwise a copy with them scrubbed (message, `code`
@@ -192,12 +238,67 @@ export declare class RequestEngine {
192
238
  * becomes a rejection.
193
239
  */
194
240
  private callTransport;
241
+ /**
242
+ * The URL a redirect leads to as messages and errors show it: the Location with the base URL's
243
+ * credentials scrubbed as sent, before resolving percent-encodes what is left of them in a new
244
+ * spelling (`'` as `%27` in a query), then resolved against the URL shown so far, its userinfo
245
+ * left out. A Location the scrub made unparseable (a credential in its host) is shown as scrubbed.
246
+ */
247
+ private shownTarget;
195
248
  /** Perform a request with Accept negotiation and transient-error retries. */
196
249
  request(method: string, path: string, options?: {
197
250
  query?: QueryParams;
198
251
  accept: string;
199
252
  }): Promise<RawResponse>;
200
- /** Perform a GET expecting JSON and parse it into `T`. */
253
+ /** Perform a GET expecting JSON and parse it into `T` (see {@link getAnswer}). */
201
254
  getJson<T>(path: string, query?: QueryParams): Promise<T>;
255
+ /**
256
+ * Perform a GET expecting JSON: the parsed value, the URL that answered (after any redirects,
257
+ * userinfo redacted) and the request as an error message names it, `GET <url>` (cut at
258
+ * MAX_MESSAGE_VALUE_LENGTH). Every error about the answer starts with that request
259
+ * (`GET <url>: the answer is not valid JSON.`), so a log of a run over many portals tells which
260
+ * one failed; the client prefixes its own envelope and shape errors with it too.
261
+ *
262
+ * The text is never changed silently: bytes that are not valid in the charset, and lone
263
+ * surrogates written as JSON escapes (`\ud800`), are replaced with U+FFFD, so the result is
264
+ * well-formed text (`jq` rejects a lone surrogate, and with it the whole output), and
265
+ * `onWarning` is told how many. One bad character never sinks a whole answer.
266
+ */
267
+ getAnswer<T>(path: string, query?: QueryParams): Promise<{
268
+ value: T;
269
+ url: string;
270
+ request: string;
271
+ }>;
202
272
  private toApiError;
203
273
  }
274
+ /** What {@link scanNumbers} found in a JSON text. */
275
+ interface NumberScan {
276
+ /** The text with each declared member's number value quoted; the input itself when there was none. */
277
+ json: string;
278
+ /** The first undeclared number literal JSON.parse cannot carry exactly, or undefined. */
279
+ inexact: string | undefined;
280
+ }
281
+ /**
282
+ * One linear pass over `json` (the text of an answer) that skips strings and looks at every
283
+ * number literal:
284
+ * - the value of a member named in `bigIntegerKeys` is quoted, as written (`"_version_":
285
+ * 1858054135406919680` becomes `"_version_":"1858054135406919680"`), so JSON.parse delivers
286
+ * it exactly as a string, whatever its size. Only a literal the JSON grammar allows is quoted,
287
+ * so a malformed one (`01`) still fails JSON.parse;
288
+ * - any other number is checked as {@link firstInexactNumber} says, and the first one JSON.parse
289
+ * cannot carry exactly is reported in `inexact`.
290
+ * Nothing is allocated per value: a member name is compared in place (only a name written with
291
+ * an escape is decoded), and the text is copied only when a declared member is found. No
292
+ * reviver, which would recurse per nesting level and overflow on deep data.
293
+ */
294
+ export declare function scanNumbers(json: string, bigIntegerKeys?: ReadonlySet<string>): NumberScan;
295
+ /**
296
+ * The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
297
+ * undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
298
+ * 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
299
+ * `9007199254740993e0`, `9007199254740992.5`), a non-zero one too small for a double
300
+ * (`1e-400` became 0), and a subnormal one that does not print as written (`3e-324` became
301
+ * `5e-324`). Below 2^53 and above the subnormal range a decimal fraction keeps a double's usual
302
+ * precision, as JSON means it. {@link scanNumbers} without declared members.
303
+ */
304
+ export declare function firstInexactNumber(json: string): string | undefined;