@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +21 -5
  2. package/dist/src/cli/commands/catalogue.d.ts +1 -2
  3. package/dist/src/cli/commands/catalogue.js +23 -15
  4. package/dist/src/cli/commands/portal.d.ts +0 -1
  5. package/dist/src/cli/commands/portal.js +13 -12
  6. package/dist/src/cli/index.d.ts +0 -1
  7. package/dist/src/cli/index.js +0 -1
  8. package/dist/src/cli/io.d.ts +2 -2
  9. package/dist/src/cli/io.js +6 -3
  10. package/dist/src/cli/program.d.ts +0 -1
  11. package/dist/src/cli/program.js +17 -11
  12. package/dist/src/cli/run.d.ts +27 -1
  13. package/dist/src/cli/run.js +90 -2
  14. package/dist/src/cli/shared.d.ts +18 -15
  15. package/dist/src/cli/shared.js +45 -54
  16. package/dist/src/client/client.d.ts +30 -9
  17. package/dist/src/client/client.js +154 -37
  18. package/dist/src/client/engine.d.ts +74 -13
  19. package/dist/src/client/engine.js +341 -71
  20. package/dist/src/client/errors.d.ts +44 -2
  21. package/dist/src/client/errors.js +94 -6
  22. package/dist/src/client/http.d.ts +23 -1
  23. package/dist/src/client/http.js +54 -33
  24. package/dist/src/client/index.d.ts +7 -6
  25. package/dist/src/client/index.js +5 -5
  26. package/dist/src/client/portals-list.d.ts +0 -1
  27. package/dist/src/client/portals-list.js +0 -1
  28. package/dist/src/client/portals.d.ts +35 -2
  29. package/dist/src/client/portals.js +86 -6
  30. package/dist/src/client/query.d.ts +0 -1
  31. package/dist/src/client/query.js +0 -1
  32. package/dist/src/client/types.d.ts +0 -1
  33. package/dist/src/client/types.js +0 -1
  34. package/dist/src/client/validate.d.ts +67 -0
  35. package/dist/src/client/validate.js +165 -0
  36. package/dist/src/index.d.ts +0 -1
  37. package/dist/src/index.js +0 -1
  38. package/package.json +4 -3
  39. package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
  40. package/dist/src/cli/commands/catalogue.js.map +0 -1
  41. package/dist/src/cli/commands/portal.d.ts.map +0 -1
  42. package/dist/src/cli/commands/portal.js.map +0 -1
  43. package/dist/src/cli/index.d.ts.map +0 -1
  44. package/dist/src/cli/index.js.map +0 -1
  45. package/dist/src/cli/io.d.ts.map +0 -1
  46. package/dist/src/cli/io.js.map +0 -1
  47. package/dist/src/cli/program.d.ts.map +0 -1
  48. package/dist/src/cli/program.js.map +0 -1
  49. package/dist/src/cli/run.d.ts.map +0 -1
  50. package/dist/src/cli/run.js.map +0 -1
  51. package/dist/src/cli/shared.d.ts.map +0 -1
  52. package/dist/src/cli/shared.js.map +0 -1
  53. package/dist/src/client/client.d.ts.map +0 -1
  54. package/dist/src/client/client.js.map +0 -1
  55. package/dist/src/client/engine.d.ts.map +0 -1
  56. package/dist/src/client/engine.js.map +0 -1
  57. package/dist/src/client/errors.d.ts.map +0 -1
  58. package/dist/src/client/errors.js.map +0 -1
  59. package/dist/src/client/http.d.ts.map +0 -1
  60. package/dist/src/client/http.js.map +0 -1
  61. package/dist/src/client/index.d.ts.map +0 -1
  62. package/dist/src/client/index.js.map +0 -1
  63. package/dist/src/client/portals-list.d.ts.map +0 -1
  64. package/dist/src/client/portals-list.js.map +0 -1
  65. package/dist/src/client/portals.d.ts.map +0 -1
  66. package/dist/src/client/portals.js.map +0 -1
  67. package/dist/src/client/query.d.ts.map +0 -1
  68. package/dist/src/client/query.js.map +0 -1
  69. package/dist/src/client/types.d.ts.map +0 -1
  70. package/dist/src/client/types.js.map +0 -1
  71. package/dist/src/index.d.ts.map +0 -1
  72. package/dist/src/index.js.map +0 -1
@@ -4,6 +4,7 @@ import { InvalidArgumentError } from "commander";
4
4
  import { isBidiControl } from "../client/engine.js";
5
5
  import { CkanError } from "../client/errors.js";
6
6
  import { findPortal } from "../client/portals.js";
7
+ import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
7
8
  import { PORTALS } from "../client/portals-list.js";
8
9
  /** commander value-parser: a non-negative integer. */
9
10
  export function parseIntArg(value) {
@@ -14,41 +15,32 @@ export function parseIntArg(value) {
14
15
  throw new InvalidArgumentError("Expected a non-negative integer.");
15
16
  }
16
17
  const n = Number(value);
17
- // Reject magnitudes that cannot be represented exactly (silent precision loss).
18
- if (!Number.isSafeInteger(n)) {
19
- throw new InvalidArgumentError("Expected a non-negative integer.");
20
- }
18
+ // The library's count rule; it also rejects magnitudes that cannot be
19
+ // represented exactly (silent precision loss).
20
+ const reason = countProblem(n);
21
+ if (reason !== undefined)
22
+ throw new InvalidArgumentError(reason);
21
23
  return n;
22
24
  }
23
25
  /**
24
- * commander value-parser: a value that is not blank. A blank filter would
25
- * otherwise be dropped and the command would silently run unfiltered.
26
+ * commander value-parser: a value that is not blank (the library's blankProblem
27
+ * rule, reported as a usage error before the client is built).
26
28
  */
27
29
  export function parseNonEmpty(value) {
28
- if (value.trim() === "") {
29
- throw new InvalidArgumentError("Expected a non-empty value.");
30
- }
30
+ const reason = blankProblem(value);
31
+ if (reason !== undefined)
32
+ throw new InvalidArgumentError(reason);
31
33
  return value;
32
34
  }
33
35
  /**
34
- * commander value-parser for a value that ends up in an HTTP header (`--user-agent`).
35
- * Node's HTTP layer throws an opaque "Invalid character in header content" at request
36
- * time for a CR/LF (or any other C0 control or DEL) and for any character above
37
- * U+00FF, which surfaced as "Unexpected error". Reject those here as a usage error,
38
- * along with a blank value. Tab is allowed, as in HTTP. Checked by char code so the
39
- * source stays free of control bytes.
36
+ * commander value-parser for a value that ends up in an HTTP header (`--user-agent`):
37
+ * the library's headerValueProblem rule (not blank, no control characters but tab,
38
+ * nothing above U+00FF), reported as a usage error.
40
39
  */
41
40
  export function parseHeaderValue(value) {
42
- parseNonEmpty(value);
43
- for (let i = 0; i < value.length; i++) {
44
- const c = value.charCodeAt(i);
45
- if ((c < 0x20 && c !== 0x09) || c === 0x7f) {
46
- throw new InvalidArgumentError("Value contains control characters.");
47
- }
48
- if (c > 0xff) {
49
- throw new InvalidArgumentError("Value contains characters outside Latin-1 (above U+00FF).");
50
- }
51
- }
41
+ const reason = headerValueProblem(value);
42
+ if (reason !== undefined)
43
+ throw new InvalidArgumentError(reason);
52
44
  return value;
53
45
  }
54
46
  /**
@@ -65,47 +57,47 @@ export function parseOptionValue(value) {
65
57
  }
66
58
  return parseNonEmpty(value);
67
59
  }
60
+ /**
61
+ * Wrap a commander value-parser for a single-valued option so that a second
62
+ * occurrence is a usage error. commander otherwise keeps only the last value, so
63
+ * `--rows 5 --rows 50` (or a second `--base-url`) silently dropped the first. The
64
+ * wrapper keeps its own count, so it also works for an option with a default (which
65
+ * commander hands the parser as `previous`); build it per program (`buildProgram`).
66
+ */
67
+ export function once(parse) {
68
+ let seen = false;
69
+ return (value) => {
70
+ if (seen)
71
+ throw new InvalidArgumentError("Given more than once; this option takes a single value.");
72
+ seen = true;
73
+ return parse(value);
74
+ };
75
+ }
68
76
  /** commander accumulator for a repeatable option whose values are checked by parseOptionValue. */
69
77
  export function collectNonEmpty(value, previous = []) {
70
78
  return previous.concat([parseOptionValue(value)]);
71
79
  }
72
- /** Build a commander value-parser for a non-negative integer within [min, max]. */
80
+ /** Build a commander value-parser for a non-negative integer within [min, max] (the library's intRangeProblem). */
73
81
  export function parseBoundedInt(min, max) {
82
+ const problem = intRangeProblem(min, max);
74
83
  return (value) => {
75
84
  const n = parseIntArg(value);
76
- if (n < min)
77
- throw new InvalidArgumentError(`Must be >= ${min}.`);
78
- if (n > max)
79
- throw new InvalidArgumentError(`Must be <= ${max}.`);
85
+ const reason = problem(n);
86
+ if (reason !== undefined)
87
+ throw new InvalidArgumentError(reason);
80
88
  return n;
81
89
  };
82
90
  }
83
91
  /**
84
- * commander value-parser for --base-url: reject a non-http(s) scheme at parse
85
- * time so `file:`, `ftp:`, etc. fail fast with a usage error rather than only
86
- * being caught later in the transport. The default transport also re-checks the
87
- * scheme per hop (and RequestEngine.buildUrl guards it for custom transports),
88
- * but this surfaces the mistake up front and independently of the transport.
92
+ * commander value-parser for --base-url (and CKAN_BASE_URL, via the preAction hook
93
+ * in program.ts): the library's baseUrlProblem rule (not blank, no whitespace, an
94
+ * absolute http(s) URL, no query string or fragment), reported as a usage error
95
+ * before the client is built.
89
96
  */
90
97
  export function parseBaseUrl(value) {
91
- let url;
92
- try {
93
- url = new URL(value);
94
- }
95
- catch {
96
- throw new InvalidArgumentError("Expected an absolute http(s) URL.");
97
- }
98
- if (url.protocol !== "http:" && url.protocol !== "https:") {
99
- throw new InvalidArgumentError(`Unsupported scheme "${url.protocol}". Expected an http(s) URL.`);
100
- }
101
- if (/[?#]/.test(value)) {
102
- throw new InvalidArgumentError("Expected a site URL without a query string or fragment.");
103
- }
104
- // `new URL` trims, but the value is used as given: a trailing space would land in
105
- // the request path (`/404%20/api/3/...`) or the host.
106
- if (value !== value.trim()) {
107
- throw new InvalidArgumentError("A base URL cannot have surrounding whitespace.");
108
- }
98
+ const reason = baseUrlProblem(value);
99
+ if (reason !== undefined)
100
+ throw new InvalidArgumentError(reason);
109
101
  return value;
110
102
  }
111
103
  /** commander value-parser for --portal: the URL of a portal in the built-in list. */
@@ -194,4 +186,3 @@ export function action(deps, fn) {
194
186
  await fn({ client, global, opts: command.opts() }, positionals);
195
187
  };
196
188
  }
197
- //# sourceMappingURL=shared.js.map
@@ -1,17 +1,29 @@
1
1
  import { type EngineOptions } from "./engine.js";
2
2
  import type { QueryParams } from "./query.js";
3
3
  import type { Group, GroupListParams, JsonValue, License, ListParams, Organization, Package, PackageSearchParams, PackageSearchResult, Resource, Status, TagListParams } from "./types.js";
4
+ /**
5
+ * Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
6
+ * of 25). Real CKAN portals list a few hundred organizations or groups at most; a
7
+ * server whose pages keep returning new entries (a broken or hostile one) would
8
+ * otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
9
+ */
10
+ export declare const MAX_ALL_FIELDS_PAGES = 400;
11
+ /**
12
+ * Pause between two `all_fields` pages, in milliseconds. A long list costs a few
13
+ * tenths of a second more; a server that never ends the list gets 10 requests a
14
+ * second at most instead of thousands.
15
+ */
16
+ export declare const ALL_FIELDS_PAGE_DELAY_MS = 100;
4
17
  /**
5
18
  * The site root of a CKAN instance, from what a user is likely to paste: portals
6
19
  * document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
7
- * dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it. A
8
- * query string or fragment is refused.
20
+ * dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
21
+ * A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
22
+ * an http(s) URL, a query string or fragment) throws CkanValidationError.
9
23
  */
10
24
  export declare function siteRoot(baseUrl: string): string;
11
25
  export declare class CkanClient {
12
- private readonly engine;
13
- /** The site this client talks to, for error messages. */
14
- private readonly site;
26
+ #private;
15
27
  constructor(options?: EngineOptions);
16
28
  /** Call any CKAN action by name and return its unwrapped `result`. */
17
29
  action<T = JsonValue>(name: string, params?: QueryParams): Promise<T>;
@@ -31,7 +43,8 @@ export declare class CkanClient {
31
43
  * several as CKAN's `fq_list` (each its own Solr filter query; all must
32
44
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
33
45
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
34
- * expects in `facet.field`.
46
+ * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
47
+ * refused (CkanValidationError), never sent or dropped.
35
48
  */
36
49
  packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
37
50
  /** A single dataset by id or name. */
@@ -42,7 +55,7 @@ export declare class CkanClient {
42
55
  groupShow(id: string): Promise<Group>;
43
56
  /** A single resource (distribution) by id. */
44
57
  resourceShow(id: string): Promise<Resource>;
45
- /** Dataset names, paged with limit/offset (a positive limit; omit it for all). */
58
+ /** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
46
59
  packageList(params?: ListParams): Promise<string[]>;
47
60
  /**
48
61
  * Organization names, or full objects with `all_fields`. See `groupOrOrgList`
@@ -51,7 +64,7 @@ export declare class CkanClient {
51
64
  organizationList(params?: GroupListParams): Promise<JsonValue[]>;
52
65
  /** Group names, or full objects with `all_fields` (paged like organizationList). */
53
66
  groupList(params?: GroupListParams): Promise<JsonValue[]>;
54
- /** Tags, optionally only those containing a substring. */
67
+ /** Tags, optionally only those containing a substring (a blank one is refused). */
55
68
  tagList(params?: TagListParams): Promise<string[]>;
56
69
  /** The licences this portal offers. */
57
70
  licenseList(): Promise<License[]>;
@@ -66,6 +79,15 @@ export declare class CkanClient {
66
79
  * some entries), so only an empty page ends the list. Entries are deduplicated
67
80
  * (by `id`), and a page that adds nothing new ends the loop too, so a server
68
81
  * that ignores `offset` cannot keep it going.
82
+ *
83
+ * Berlin counts hidden entries in its `limit`/`offset` window but leaves them
84
+ * out of the answer, so a window smaller than a full page (the last one before
85
+ * `limit` is reached) can come back empty although more entries follow. Such a
86
+ * miss is asked again at the same offset with a full page before it ends the list.
87
+ *
88
+ * Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
89
+ * `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
90
+ * keep returning new entries) the call fails with a CkanParseError.
69
91
  */
70
92
  private groupOrOrgList;
71
93
  /**
@@ -74,4 +96,3 @@ export declare class CkanClient {
74
96
  */
75
97
  private show;
76
98
  }
77
- //# sourceMappingURL=client.d.ts.map
@@ -1,7 +1,8 @@
1
1
  // CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
2
2
  // Action API (`<site>/api/3/action`).
3
- import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText } from "./engine.js";
4
- import { CkanError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
3
+ import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
4
+ import { CkanError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
5
+ import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
5
6
  const ACTION = "/api/3/action";
6
7
  /**
7
8
  * CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
@@ -12,6 +13,20 @@ const ACTION = "/api/3/action";
12
13
  const ACTION_NAME = /^[a-z0-9_]+$/;
13
14
  /** CKAN's default cap on an `all_fields` organization/group list. */
14
15
  const ALL_FIELDS_PAGE = 25;
16
+ /**
17
+ * Most pages one `all_fields` list call requests (10,000 entries at CKAN's page size
18
+ * of 25). Real CKAN portals list a few hundred organizations or groups at most; a
19
+ * server whose pages keep returning new entries (a broken or hostile one) would
20
+ * otherwise be paged forever. Past it the call fails with a CkanParseError naming it.
21
+ */
22
+ export const MAX_ALL_FIELDS_PAGES = 400;
23
+ /**
24
+ * Pause between two `all_fields` pages, in milliseconds. A long list costs a few
25
+ * tenths of a second more; a server that never ends the list gets 10 requests a
26
+ * second at most instead of thousands.
27
+ */
28
+ export const ALL_FIELDS_PAGE_DELAY_MS = 100;
29
+ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
15
30
  /** A key for a list entry — its `id`, else its JSON — to drop duplicates across pages. */
16
31
  function entryKey(entry) {
17
32
  if (typeof entry === "object" && entry !== null && !Array.isArray(entry)) {
@@ -22,21 +37,60 @@ function entryKey(entry) {
22
37
  return JSON.stringify(entry);
23
38
  }
24
39
  /**
25
- * Drop undefined (and empty-string) values so only the parameters the caller
26
- * actually set are sent. An empty string filter (e.g. `tags --query ""`) is
27
- * treated as "no filter" rather than forwarded as `query=`.
40
+ * Drop undefined values so only the parameters the caller actually set are sent,
41
+ * and refuse a blank parameter name, a blank value or a list with a blank entry
42
+ * (CkanValidationError). CKAN reads an empty parameter as "not given", so a blank
43
+ * filter such as `tagList({ query: "" })` would silently return everything.
28
44
  */
29
45
  function prune(params) {
46
+ assertParams(params);
30
47
  // A null-prototype object, so a `__proto__` key is kept as a parameter instead
31
48
  // of setting the prototype (and being lost).
32
49
  const out = Object.create(null);
33
50
  for (const [k, v] of Object.entries(params)) {
34
- if (v === undefined || v === "")
51
+ if (v === undefined)
35
52
  continue;
36
- out[k] = v;
53
+ assertValid("parameter name", k, blankProblem);
54
+ // A value the query string can't carry (an object, NaN, a function) is refused,
55
+ // not sent as "[object Object]" or "NaN".
56
+ out[k] = assertValid(k, v, queryValueProblem);
37
57
  }
38
58
  return out;
39
59
  }
60
+ /**
61
+ * A method's parameter object: a plain object (or undefined, for the defaults).
62
+ * `null`, a string or an array would otherwise fail as a raw TypeError.
63
+ */
64
+ function assertParams(params) {
65
+ if (params === undefined)
66
+ return;
67
+ if (typeof params !== "object" || params === null || Array.isArray(params)) {
68
+ throw new CkanValidationError("Invalid params: Expected an object.");
69
+ }
70
+ }
71
+ /**
72
+ * Throw CkanValidationError unless every own key of `params` is in `known`. CKAN
73
+ * ignores a parameter it doesn't know, so a misspelled `fqs` or a JSON `__proto__`
74
+ * key would run the search unfiltered over the whole catalogue. Any other CKAN
75
+ * parameter goes through `action(name, params)`, which sends every key.
76
+ */
77
+ function assertKeys(method, action, params, known) {
78
+ for (const key of Object.keys(params)) {
79
+ if (!known.includes(key)) {
80
+ throw new CkanValidationError(`Invalid ${method} parameter ${JSON.stringify(cutForMessage(key))}: not a parameter of ${method}. ` +
81
+ `Known: ${known.join(", ")}. Use action("${action}", params) to send another CKAN parameter.`);
82
+ }
83
+ }
84
+ }
85
+ const SEARCH_KEYS = ["q", "fq", "rows", "start", "sort", "facet_field", "facet_limit"];
86
+ const PACKAGE_LIST_KEYS = ["limit", "offset"];
87
+ const GROUP_LIST_KEYS = ["limit", "offset", "all_fields"];
88
+ const TAG_LIST_KEYS = ["query"];
89
+ /** Check an optional text parameter (`q`, `sort`, a tag query): a non-blank string. */
90
+ function assertText(name, value) {
91
+ if (value !== undefined)
92
+ assertValid(name, value, textProblem);
93
+ }
40
94
  /** A JSON object (not null, not an array). */
41
95
  function isObject(value) {
42
96
  return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -62,46 +116,55 @@ function assertLimit(limit) {
62
116
  if (limit === undefined)
63
117
  return;
64
118
  if (!Number.isSafeInteger(limit) || limit < 1) {
65
- throw new CkanError(`Invalid limit: expected a positive integer, got ${String(limit)}. Leave it out for the whole list.`);
119
+ throw new CkanValidationError(`Invalid limit: expected a positive integer, got ${String(limit)}. Leave it out for the whole list.`);
66
120
  }
67
121
  }
122
+ /** Check an optional count or offset (`rows`, `start`, `offset`): see countProblem. */
123
+ function assertCount(name, value) {
124
+ if (value !== undefined)
125
+ assertValid(name, value, countProblem);
126
+ }
68
127
  /**
69
128
  * The site root of a CKAN instance, from what a user is likely to paste: portals
70
129
  * document their API as `<site>/api/3/action`, so that suffix (or `/api/3`) is
71
- * dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it. A
72
- * query string or fragment is refused.
130
+ * dropped. A CKAN mounted under a sub-path (`https://host/ckan`) keeps it.
131
+ * A base URL that breaks the rules of `validateBaseUrl` (blank, whitespace, not
132
+ * an http(s) URL, a query string or fragment) throws CkanValidationError.
73
133
  */
74
134
  export function siteRoot(baseUrl) {
75
- // The API path is appended to the base URL, so a query string or fragment in it
76
- // would end up in front of `/api/3/action` and break every request.
77
- if (/[?#]/.test(baseUrl)) {
78
- throw new CkanError(`The base URL must not contain a query string or fragment: ${redactUrl(baseUrl)}`);
79
- }
80
- return baseUrl.replace(/\/+$/, "").replace(/\/api\/3(\/action)?$/, "");
135
+ // Checked before the suffix strip: a trailing space would hide the suffix from it.
136
+ return validateBaseUrl(baseUrl).replace(/\/api\/3(\/action)?$/, "");
81
137
  }
82
138
  export class CkanClient {
83
- engine;
84
- /** The site this client talks to, for error messages. */
85
- site;
139
+ // Real private fields (not TypeScript's `private`): console.log, util.inspect and
140
+ // JSON.stringify of a client never show them, so a password in the base URL can't
141
+ // be logged by accident.
142
+ #engine;
143
+ /** The site this client talks to, for error messages (always through redactUrl). */
144
+ #site;
145
+ /** The pause between `all_fields` pages (EngineOptions.sleep, for tests). */
146
+ #sleep;
86
147
  constructor(options = {}) {
87
- this.site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
88
- this.engine = new RequestEngine({ ...options, baseUrl: this.site });
148
+ this.#site = siteRoot(options.baseUrl ?? DEFAULT_BASE_URL);
149
+ // The engine checks every option, `sleep` included, before the client uses it.
150
+ this.#engine = new RequestEngine({ ...options, baseUrl: this.#site });
151
+ this.#sleep = options.sleep ?? realSleep;
89
152
  }
90
153
  /** Call any CKAN action by name and return its unwrapped `result`. */
91
154
  async action(name, params = {}) {
92
- if (!ACTION_NAME.test(name)) {
93
- throw new CkanError(`Invalid CKAN action name: "${name}"`);
155
+ if (typeof name !== "string" || !ACTION_NAME.test(name)) {
156
+ throw new CkanValidationError(`Invalid CKAN action name: "${cutForMessage(typeof name === "string" ? name : String(name))}"`);
94
157
  }
95
- const env = await this.engine.getJson(`${ACTION}/${name}`, prune(params));
158
+ const env = await this.#engine.getJson(`${ACTION}/${name}`, prune(params));
96
159
  // Another JSON API at the same path (or a proxy's JSON error page) is not an
97
160
  // envelope; say so rather than "failed: undefined".
98
161
  if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
99
- throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.site)} a CKAN site?`);
162
+ throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
100
163
  }
101
164
  if (!env.success) {
102
165
  // A success:false envelope can come with HTTP 200, so it never passes the
103
166
  // engine's error-detail sanitising: strip terminal controls here too.
104
- throw new CkanError(`CKAN action "${name}" failed: ${sanitizeServerText(describeCkanError(env.error))}`);
167
+ throw new CkanError(`CKAN action "${name}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`);
105
168
  }
106
169
  // `{"success": true}` without a result would print nothing useful (and the CLI
107
170
  // would crash rendering `undefined`); CKAN always sends one, `null` included.
@@ -132,10 +195,23 @@ export class CkanClient {
132
195
  * several as CKAN's `fq_list` (each its own Solr filter query; all must
133
196
  * match). `fq_list` is only used for two or more, because CKAN splits a lone
134
197
  * `fq_list` value into characters. Facet fields go out as the JSON list CKAN
135
- * expects in `facet.field`.
198
+ * expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
199
+ * refused (CkanValidationError), never sent or dropped.
136
200
  */
137
- packageSearch(params = {}) {
138
- const fq = (params.fq ?? []).filter((f) => f !== "");
201
+ async packageSearch(params = {}) {
202
+ assertParams(params);
203
+ assertKeys("packageSearch", "package_search", params, SEARCH_KEYS);
204
+ assertText("q", params.q);
205
+ if (params.fq !== undefined)
206
+ assertValid("fq", params.fq, textListProblem);
207
+ assertText("sort", params.sort);
208
+ if (params.facet_field !== undefined)
209
+ assertValid("facet_field", params.facet_field, textListProblem);
210
+ assertCount("rows", params.rows);
211
+ assertCount("start", params.start);
212
+ if (params.facet_limit !== undefined)
213
+ assertValid("facet_limit", params.facet_limit, facetLimitProblem);
214
+ const fq = params.fq ?? [];
139
215
  const facetFields = params.facet_field ?? [];
140
216
  return this.typed("package_search", {
141
217
  q: params.q,
@@ -164,9 +240,12 @@ export class CkanClient {
164
240
  resourceShow(id) {
165
241
  return this.show("resource_show", id);
166
242
  }
167
- /** Dataset names, paged with limit/offset (a positive limit; omit it for all). */
243
+ /** Dataset names, paged with limit/offset (a positive limit, omit it for all; a non-negative offset). */
168
244
  async packageList(params = {}) {
245
+ assertParams(params);
246
+ assertKeys("packageList", "package_list", params, PACKAGE_LIST_KEYS);
169
247
  assertLimit(params.limit);
248
+ assertCount("offset", params.offset);
170
249
  return this.typed("package_list", { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
171
250
  }
172
251
  /**
@@ -180,8 +259,11 @@ export class CkanClient {
180
259
  groupList(params = {}) {
181
260
  return this.groupOrOrgList("group_list", params);
182
261
  }
183
- /** Tags, optionally only those containing a substring. */
184
- tagList(params = {}) {
262
+ /** Tags, optionally only those containing a substring (a blank one is refused). */
263
+ async tagList(params = {}) {
264
+ assertParams(params);
265
+ assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
266
+ assertText("query", params.query);
185
267
  return this.typed("tag_list", { query: params.query }, Array.isArray, "an array");
186
268
  }
187
269
  /** The licences this portal offers. */
@@ -199,9 +281,25 @@ export class CkanClient {
199
281
  * some entries), so only an empty page ends the list. Entries are deduplicated
200
282
  * (by `id`), and a page that adds nothing new ends the loop too, so a server
201
283
  * that ignores `offset` cannot keep it going.
284
+ *
285
+ * Berlin counts hidden entries in its `limit`/`offset` window but leaves them
286
+ * out of the answer, so a window smaller than a full page (the last one before
287
+ * `limit` is reached) can come back empty although more entries follow. Such a
288
+ * miss is asked again at the same offset with a full page before it ends the list.
289
+ *
290
+ * Bounds: pages are `ALL_FIELDS_PAGE_DELAY_MS` apart, and after
291
+ * `MAX_ALL_FIELDS_PAGES` pages without reaching the end (a server whose pages
292
+ * keep returning new entries) the call fails with a CkanParseError.
202
293
  */
203
294
  async groupOrOrgList(action, params) {
295
+ assertParams(params);
296
+ assertKeys(action === "group_list" ? "groupList" : "organizationList", action, params, GROUP_LIST_KEYS);
297
+ if (params.all_fields !== undefined && typeof params.all_fields !== "boolean") {
298
+ throw new CkanValidationError("Invalid all_fields: Expected a boolean.");
299
+ }
204
300
  assertLimit(params.limit);
301
+ // Before the first request, and before the all_fields pager steps on from it.
302
+ assertCount("offset", params.offset);
205
303
  if (!params.all_fields) {
206
304
  return this.typed(action, { limit: params.limit, offset: params.offset }, Array.isArray, "an array");
207
305
  }
@@ -209,12 +307,25 @@ export class CkanClient {
209
307
  let offset = params.offset ?? 0;
210
308
  const seen = new Set();
211
309
  const out = [];
310
+ let pages = 0;
311
+ // Set after a short window came back empty (or with nothing new): from then on
312
+ // every page is a full one.
313
+ let fullPages = false;
212
314
  while (out.length < wanted) {
315
+ if (pages >= MAX_ALL_FIELDS_PAGES) {
316
+ throw new CkanParseError(`${action} with all_fields: stopped after ${MAX_ALL_FIELDS_PAGES} pages (MAX_ALL_FIELDS_PAGES) without ` +
317
+ `reaching the end of the list; the server keeps returning new entries, which no CKAN list has. ` +
318
+ `Pass a limit (--limit) to fetch part of it.`);
319
+ }
320
+ if (pages > 0)
321
+ await this.#sleep(ALL_FIELDS_PAGE_DELAY_MS);
322
+ const limit = fullPages ? ALL_FIELDS_PAGE : Math.min(ALL_FIELDS_PAGE, wanted - out.length);
213
323
  const page = await this.typed(action, {
214
324
  all_fields: true,
215
- limit: Math.min(ALL_FIELDS_PAGE, wanted - out.length),
325
+ limit,
216
326
  offset: offset === 0 ? undefined : offset,
217
327
  }, Array.isArray, "an array");
328
+ pages += 1;
218
329
  let added = 0;
219
330
  for (const entry of page) {
220
331
  const key = entryKey(entry);
@@ -224,8 +335,15 @@ export class CkanClient {
224
335
  out.push(entry);
225
336
  added += 1;
226
337
  }
227
- if (page.length === 0 || added === 0)
338
+ if (page.length === 0 || added === 0) {
339
+ // A short window may have held only hidden entries: ask once more, same
340
+ // offset, for a full page before taking it as the end.
341
+ if (limit < ALL_FIELDS_PAGE) {
342
+ fullPages = true;
343
+ continue;
344
+ }
228
345
  break;
346
+ }
229
347
  offset += page.length;
230
348
  }
231
349
  return out.slice(0, wanted);
@@ -235,9 +353,8 @@ export class CkanClient {
235
353
  * CKAN would answer with a 409 validation error instead.
236
354
  */
237
355
  async show(action, id) {
238
- if (id.trim() === "")
239
- throw new CkanError(`${action} needs an id or name.`);
356
+ if (typeof id !== "string" || id.trim() === "")
357
+ throw new CkanValidationError(`${action} needs an id or name.`);
240
358
  return this.typed(action, { id }, isObject, "a JSON object");
241
359
  }
242
360
  }
243
- //# sourceMappingURL=client.js.map