@maschinenlesbar.org/ckan-cli 0.0.6 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/README.md +21 -5
  2. package/dist/src/cli/commands/catalogue.d.ts +1 -2
  3. package/dist/src/cli/commands/catalogue.js +23 -15
  4. package/dist/src/cli/commands/portal.d.ts +0 -1
  5. package/dist/src/cli/commands/portal.js +13 -12
  6. package/dist/src/cli/index.d.ts +0 -1
  7. package/dist/src/cli/index.js +0 -1
  8. package/dist/src/cli/io.d.ts +2 -2
  9. package/dist/src/cli/io.js +6 -3
  10. package/dist/src/cli/program.d.ts +0 -1
  11. package/dist/src/cli/program.js +17 -11
  12. package/dist/src/cli/run.d.ts +27 -1
  13. package/dist/src/cli/run.js +90 -2
  14. package/dist/src/cli/shared.d.ts +18 -15
  15. package/dist/src/cli/shared.js +45 -54
  16. package/dist/src/client/client.d.ts +30 -9
  17. package/dist/src/client/client.js +154 -37
  18. package/dist/src/client/engine.d.ts +74 -13
  19. package/dist/src/client/engine.js +341 -71
  20. package/dist/src/client/errors.d.ts +44 -2
  21. package/dist/src/client/errors.js +94 -6
  22. package/dist/src/client/http.d.ts +23 -1
  23. package/dist/src/client/http.js +54 -33
  24. package/dist/src/client/index.d.ts +7 -6
  25. package/dist/src/client/index.js +5 -5
  26. package/dist/src/client/portals-list.d.ts +0 -1
  27. package/dist/src/client/portals-list.js +0 -1
  28. package/dist/src/client/portals.d.ts +35 -2
  29. package/dist/src/client/portals.js +86 -6
  30. package/dist/src/client/query.d.ts +0 -1
  31. package/dist/src/client/query.js +0 -1
  32. package/dist/src/client/types.d.ts +0 -1
  33. package/dist/src/client/types.js +0 -1
  34. package/dist/src/client/validate.d.ts +67 -0
  35. package/dist/src/client/validate.js +165 -0
  36. package/dist/src/index.d.ts +0 -1
  37. package/dist/src/index.js +0 -1
  38. package/package.json +4 -3
  39. package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
  40. package/dist/src/cli/commands/catalogue.js.map +0 -1
  41. package/dist/src/cli/commands/portal.d.ts.map +0 -1
  42. package/dist/src/cli/commands/portal.js.map +0 -1
  43. package/dist/src/cli/index.d.ts.map +0 -1
  44. package/dist/src/cli/index.js.map +0 -1
  45. package/dist/src/cli/io.d.ts.map +0 -1
  46. package/dist/src/cli/io.js.map +0 -1
  47. package/dist/src/cli/program.d.ts.map +0 -1
  48. package/dist/src/cli/program.js.map +0 -1
  49. package/dist/src/cli/run.d.ts.map +0 -1
  50. package/dist/src/cli/run.js.map +0 -1
  51. package/dist/src/cli/shared.d.ts.map +0 -1
  52. package/dist/src/cli/shared.js.map +0 -1
  53. package/dist/src/client/client.d.ts.map +0 -1
  54. package/dist/src/client/client.js.map +0 -1
  55. package/dist/src/client/engine.d.ts.map +0 -1
  56. package/dist/src/client/engine.js.map +0 -1
  57. package/dist/src/client/errors.d.ts.map +0 -1
  58. package/dist/src/client/errors.js.map +0 -1
  59. package/dist/src/client/http.d.ts.map +0 -1
  60. package/dist/src/client/http.js.map +0 -1
  61. package/dist/src/client/index.d.ts.map +0 -1
  62. package/dist/src/client/index.js.map +0 -1
  63. package/dist/src/client/portals-list.d.ts.map +0 -1
  64. package/dist/src/client/portals-list.js.map +0 -1
  65. package/dist/src/client/portals.d.ts.map +0 -1
  66. package/dist/src/client/portals.js.map +0 -1
  67. package/dist/src/client/query.d.ts.map +0 -1
  68. package/dist/src/client/query.js.map +0 -1
  69. package/dist/src/client/types.d.ts.map +0 -1
  70. package/dist/src/client/types.js.map +0 -1
  71. package/dist/src/index.d.ts.map +0 -1
  72. package/dist/src/index.js.map +0 -1
@@ -1,9 +1,34 @@
1
1
  /**
2
2
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
3
3
  * credential in a base URL never reaches an error message, a log or CI output.
4
- * A URL without userinfo, or one that does not parse, is returned unchanged.
4
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
5
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
6
+ * value without userinfo is returned unchanged.
5
7
  */
6
8
  export declare function redactUrl(url: string): string;
9
+ /**
10
+ * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
11
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
12
+ * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
13
+ * is everything between `://` and the last `@` before the host. A value without a scheme
14
+ * counts when it reads `user:password@host`. Used to redact those exact strings from text
15
+ * that echoes the value (usage errors, help), whatever characters the password contains.
16
+ */
17
+ export declare function credentialsIn(value: string): string[];
18
+ /**
19
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
20
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
21
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
22
+ */
23
+ export declare function redactCredentials(text: string, credentials: readonly string[]): string;
24
+ /**
25
+ * Longest URL or server text (in characters) an error message shows. A long search
26
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
27
+ * log. The error's `url`, `detail` and `body` properties keep the full value.
28
+ */
29
+ export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
30
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
31
+ export declare function cutForMessage(text: string): string;
7
32
  /** Base class for every error originating from this client. */
8
33
  export declare class CkanError extends Error {
9
34
  constructor(message: string, options?: {
@@ -20,12 +45,23 @@ export declare class CkanApiError extends CkanError {
20
45
  readonly url: string;
21
46
  readonly method: string;
22
47
  readonly body: string;
48
+ /** How many times the engine retried the request before giving up (0 when it did not). */
49
+ readonly retries: number;
50
+ /**
51
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
52
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
53
+ * else undefined.
54
+ */
55
+ readonly retryAfterMs: number | undefined;
23
56
  constructor(args: {
24
57
  status: number;
25
58
  url: string;
26
59
  method: string;
27
60
  body: string;
28
61
  detail?: string;
62
+ retries?: number;
63
+ retryAfterMs?: number;
64
+ maxRetryAfterMs?: number;
29
65
  });
30
66
  /** True for statuses the API documents as transient and retry-able. */
31
67
  get isRetryable(): boolean;
@@ -36,6 +72,13 @@ export declare class CkanNetworkError extends CkanError {
36
72
  /** The response body could not be parsed as the expected JSON shape. */
37
73
  export declare class CkanParseError extends CkanError {
38
74
  }
75
+ /**
76
+ * An input the library refuses before sending any request: a client option or a
77
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
78
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
79
+ */
80
+ export declare class CkanValidationError extends CkanError {
81
+ }
39
82
  /**
40
83
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
41
84
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -43,4 +86,3 @@ export declare class CkanParseError extends CkanError {
43
86
  * (`{"__type": "Validation Error", "rows": ["Invalid integer"]}`).
44
87
  */
45
88
  export declare function describeCkanError(error: unknown): string;
46
- //# sourceMappingURL=errors.d.ts.map
@@ -3,7 +3,9 @@
3
3
  /**
4
4
  * Replace the userinfo of a URL (`https://user:secret@host/...`) with `***`, so a
5
5
  * credential in a base URL never reaches an error message, a log or CI output.
6
- * A URL without userinfo, or one that does not parse, is returned unchanged.
6
+ * A value that does not parse as a URL (a port typo, an unencoded `#` in the
7
+ * password, no scheme) has its userinfo cut out by text (`credentialsIn`); a
8
+ * value without userinfo is returned unchanged.
7
9
  */
8
10
  export function redactUrl(url) {
9
11
  let parsed;
@@ -11,14 +13,70 @@ export function redactUrl(url) {
11
13
  parsed = new URL(url);
12
14
  }
13
15
  catch {
14
- return url;
16
+ // A value that doesn't parse can still carry credentials: cut them out by text.
17
+ return redactCredentials(url, credentialsIn(url));
15
18
  }
19
+ // `user:pw@host` without a scheme parses as a URL with the scheme "user:": no userinfo.
16
20
  if (parsed.username === "" && parsed.password === "")
17
- return url;
21
+ return redactCredentials(url, credentialsIn(url));
18
22
  parsed.username = "***";
19
23
  parsed.password = "";
20
24
  return parsed.href;
21
25
  }
26
+ /**
27
+ * The userinfo a URL-like value carries, exactly as written — `["alice:pa#ss"]` for
28
+ * `https://alice:pa#ss@host` — or `[]` when it carries none. It works on values that don't
29
+ * parse as a URL too, and on values with a prefix (`--base-url=https://u:p@h`): the userinfo
30
+ * is everything between `://` and the last `@` before the host. A value without a scheme
31
+ * counts when it reads `user:password@host`. Used to redact those exact strings from text
32
+ * that echoes the value (usage errors, help), whatever characters the password contains.
33
+ */
34
+ export function credentialsIn(value) {
35
+ const schemeAt = value.indexOf("://");
36
+ const rest = schemeAt >= 0 ? value.slice(schemeAt + 3) : value;
37
+ // Without a scheme only the unmistakable `user:password@host` form counts.
38
+ if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
39
+ return [];
40
+ // The URL itself starts at its scheme (`--base-url=https://…` has a prefix).
41
+ const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
42
+ let parses = false;
43
+ try {
44
+ new URL(schemeAt >= 0 ? value.slice(scheme?.index ?? schemeAt) : `http://${rest}`);
45
+ parses = true;
46
+ }
47
+ catch {
48
+ // Doesn't parse: the password may hold "/", "?", "#" or spaces.
49
+ }
50
+ // In a URL that parses, the userinfo ends at the last "@" of the authority (before the
51
+ // first "/", "?" or "#"); in one that doesn't, at the last "@" of the value.
52
+ const authority = parses ? rest.slice(0, rest.search(/[/?#]|$/)) : rest;
53
+ const end = authority.lastIndexOf("@");
54
+ return end > 0 ? [rest.slice(0, end)] : [];
55
+ }
56
+ /**
57
+ * `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
58
+ * followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
59
+ * passwords with spaces, quotes, `#`, `?` or `/` that no URL pattern can delimit.
60
+ */
61
+ export function redactCredentials(text, credentials) {
62
+ let out = text;
63
+ for (const secret of credentials) {
64
+ if (secret === "")
65
+ continue;
66
+ out = out.split(`${secret}@`).join("***@");
67
+ }
68
+ return out;
69
+ }
70
+ /**
71
+ * Longest URL or server text (in characters) an error message shows. A long search
72
+ * query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
73
+ * log. The error's `url`, `detail` and `body` properties keep the full value.
74
+ */
75
+ export const MAX_MESSAGE_VALUE_LENGTH = 500;
76
+ /** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
77
+ export function cutForMessage(text) {
78
+ return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text.slice(0, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
79
+ }
22
80
  /** Base class for every error originating from this client. */
23
81
  export class CkanError extends Error {
24
82
  constructor(message, options) {
@@ -36,16 +94,40 @@ export class CkanApiError extends CkanError {
36
94
  url;
37
95
  method;
38
96
  body;
97
+ /** How many times the engine retried the request before giving up (0 when it did not). */
98
+ retries;
99
+ /**
100
+ * The wait the server asked for in `Retry-After` (milliseconds) when it was longer
101
+ * than the engine waits (`MAX_RETRY_AFTER_MS`), so the request was not retried;
102
+ * else undefined.
103
+ */
104
+ retryAfterMs;
39
105
  constructor(args) {
40
106
  // The URL is shown without userinfo: a credential in --base-url must not leak.
41
107
  const url = redactUrl(args.url);
42
- const detailPart = args.detail ? `: ${args.detail}` : "";
43
- super(`HTTP ${args.status} for ${args.method} ${url}${detailPart}`);
108
+ const parts = [];
109
+ if (args.detail)
110
+ parts.push(cutForMessage(args.detail));
111
+ if (args.retryAfterMs !== undefined) {
112
+ // Say why the retries the caller asked for never ran: the server asked for a
113
+ // wait longer than the engine sleeps, and retrying earlier would land inside it.
114
+ const wait = Math.ceil(args.retryAfterMs / 1000);
115
+ const cap = args.maxRetryAfterMs === undefined ? "" : `, longer than the ${args.maxRetryAfterMs / 1000} s the client waits`;
116
+ parts.push(`the server asked to retry after ${wait} s${cap}; not retried — try again after that`);
117
+ }
118
+ const detailPart = parts.length > 0 ? `: ${parts.join("; ")}` : "";
119
+ const retries = args.retries ?? 0;
120
+ // Say that the status persisted through retries, so a user knows whether raising
121
+ // --max-retries could help.
122
+ const retryPart = retries > 0 ? ` (after ${retries} ${retries === 1 ? "retry" : "retries"})` : "";
123
+ super(`HTTP ${args.status} for ${args.method} ${cutForMessage(url)}${detailPart}${retryPart}`);
44
124
  this.status = args.status;
45
125
  this.url = url;
46
126
  this.method = args.method;
47
127
  this.body = args.body;
48
128
  this.detail = args.detail;
129
+ this.retries = retries;
130
+ this.retryAfterMs = args.retryAfterMs;
49
131
  }
50
132
  /** True for statuses the API documents as transient and retry-able. */
51
133
  get isRetryable() {
@@ -58,6 +140,13 @@ export class CkanNetworkError extends CkanError {
58
140
  /** The response body could not be parsed as the expected JSON shape. */
59
141
  export class CkanParseError extends CkanError {
60
142
  }
143
+ /**
144
+ * An input the library refuses before sending any request: a client option or a
145
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
146
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
147
+ */
148
+ export class CkanValidationError extends CkanError {
149
+ }
61
150
  /**
62
151
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
63
152
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -91,4 +180,3 @@ function solrReason(message) {
91
180
  return undefined;
92
181
  return match[1].replace(/\\+(['"])/g, "$1").trim();
93
182
  }
94
- //# sourceMappingURL=errors.js.map
@@ -10,13 +10,36 @@ export interface HttpRequest {
10
10
  timeoutMs?: number;
11
11
  /** Hard cap on the response body size in bytes; the request aborts if exceeded. */
12
12
  maxResponseBytes?: number;
13
+ /**
14
+ * Aborted when the engine's overall deadline (`timeoutMs`) passes. A transport should
15
+ * stop the request then (`fetch(url, { signal })`); the engine rejects at the deadline
16
+ * either way, and enforces `maxResponseBytes` on the body it gets back, so neither
17
+ * limit depends on the transport.
18
+ */
19
+ signal?: AbortSignal;
20
+ /**
21
+ * Always `"manual"` from the engine: a transport must not follow redirects. The
22
+ * engine follows them itself and decides per hop whether the `Authorization`
23
+ * header (the base URL's userinfo) goes along (same origin only). A fetch-based
24
+ * transport passes it on: `fetch(url, { redirect })`.
25
+ */
26
+ redirect?: "manual";
13
27
  }
14
28
  export interface HttpResponse {
15
29
  status: number;
16
30
  headers: http.IncomingHttpHeaders;
17
31
  body: Buffer;
32
+ /**
33
+ * The URL the response came from, if the transport knows it (fetch's
34
+ * `response.url`). When it is on another origin than the request, the transport
35
+ * followed a redirect itself and the engine rejects the response with a
36
+ * CkanNetworkError.
37
+ */
38
+ url?: string;
18
39
  }
19
40
  export type Transport = (request: HttpRequest) => Promise<HttpResponse>;
41
+ /** The message for a body over the size cap, naming the option on both sides. */
42
+ export declare function sizeLimitMessage(maxBytes: number): string;
20
43
  /**
21
44
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
22
45
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -28,4 +51,3 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
28
51
  * (connection errors, timeouts, malformed URLs).
29
52
  */
30
53
  export declare const nodeHttpTransport: Transport;
31
- //# sourceMappingURL=http.d.ts.map
@@ -8,6 +8,10 @@
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
10
  import { CkanNetworkError, redactUrl } from "./errors.js";
11
+ /** The message for a body over the size cap, naming the option on both sides. */
12
+ export function sizeLimitMessage(maxBytes) {
13
+ return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
14
+ }
11
15
  /**
12
16
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
13
17
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -50,40 +54,49 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
50
54
  clearTimeout(deadline);
51
55
  reject(err);
52
56
  };
53
- const req = driver.request(url, {
54
- method: request.method,
55
- headers: request.headers,
56
- }, (res) => {
57
- const chunks = [];
58
- let received = 0;
59
- let aborted = false;
60
- res.on("data", (chunk) => {
61
- if (aborted)
62
- return;
63
- received += chunk.length;
64
- if (maxBytes !== undefined && received > maxBytes) {
65
- aborted = true;
66
- res.destroy();
67
- settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
68
- return;
69
- }
70
- chunks.push(chunk);
71
- });
72
- res.on("end", () => {
73
- if (aborted)
74
- return;
75
- settleResolve({
76
- status: res.statusCode ?? 0,
77
- headers: res.headers,
78
- body: Buffer.concat(chunks),
57
+ // Node validates the headers synchronously and throws a bare TypeError
58
+ // (ERR_INVALID_CHAR) for one it cannot send; report it as a typed failure.
59
+ let req;
60
+ try {
61
+ req = driver.request(url, {
62
+ method: request.method,
63
+ headers: request.headers,
64
+ }, (res) => {
65
+ const chunks = [];
66
+ let received = 0;
67
+ let aborted = false;
68
+ res.on("data", (chunk) => {
69
+ if (aborted)
70
+ return;
71
+ received += chunk.length;
72
+ if (maxBytes !== undefined && received > maxBytes) {
73
+ aborted = true;
74
+ res.destroy();
75
+ settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
76
+ return;
77
+ }
78
+ chunks.push(chunk);
79
+ });
80
+ res.on("end", () => {
81
+ if (aborted)
82
+ return;
83
+ settleResolve({
84
+ status: res.statusCode ?? 0,
85
+ headers: res.headers,
86
+ body: Buffer.concat(chunks),
87
+ });
88
+ });
89
+ res.on("error", (err) => {
90
+ if (aborted)
91
+ return; // we already rejected with the size-cap error
92
+ settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
79
93
  });
80
94
  });
81
- res.on("error", (err) => {
82
- if (aborted)
83
- return; // we already rejected with the size-cap error
84
- settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
85
- });
86
- });
95
+ }
96
+ catch (err) {
97
+ reject(new CkanNetworkError(`Invalid request: ${err instanceof Error ? err.message : String(err)}`, { cause: err }));
98
+ return;
99
+ }
87
100
  if (request.timeoutMs && request.timeoutMs > 0) {
88
101
  // Two complementary guards, both using timeoutMs:
89
102
  // - setTimeout: a socket-*inactivity* timeout (no bytes for timeoutMs).
@@ -100,6 +113,15 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
100
113
  // Don't let a pending deadline timer keep the event loop alive on its own.
101
114
  deadline.unref?.();
102
115
  }
116
+ if (request.signal !== undefined) {
117
+ const abort = () => {
118
+ req.destroy(new CkanNetworkError(`Request exceeded the ${request.timeoutMs ?? 0}ms deadline`));
119
+ };
120
+ if (request.signal.aborted)
121
+ abort();
122
+ else
123
+ request.signal.addEventListener("abort", abort, { once: true });
124
+ }
103
125
  req.on("error", (err) => {
104
126
  // A timeout destroy already passes an CkanNetworkError; don't double-wrap.
105
127
  settleReject(err instanceof CkanNetworkError ? err : new CkanNetworkError(err.message, { cause: err }));
@@ -108,4 +130,3 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
108
130
  req.write(request.body);
109
131
  req.end();
110
132
  });
111
- //# sourceMappingURL=http.js.map
@@ -1,13 +1,14 @@
1
- export { CkanClient, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
1
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
+ export { assertValid } from "./validate.js";
10
+ export type { Problem } from "./validate.js";
9
11
  export { PORTALS } from "./portals-list.js";
10
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
11
- export type { PortalCheck } from "./portals.js";
12
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
13
+ export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
12
14
  export * from "./types.js";
13
- //# sourceMappingURL=index.d.ts.map
@@ -1,10 +1,10 @@
1
1
  // Public entry point for the API client library.
2
- export { CkanClient, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
2
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
+ export { assertValid } from "./validate.js";
7
8
  export { PORTALS } from "./portals-list.js";
8
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
9
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
9
10
  export * from "./types.js";
10
- //# sourceMappingURL=index.js.map
@@ -1,3 +1,2 @@
1
1
  import type { Portal } from "./types.js";
2
2
  export declare const PORTALS: readonly Portal[];
3
- //# sourceMappingURL=portals-list.d.ts.map
@@ -291,4 +291,3 @@ export const PORTALS = frozen([
291
291
  "note": "Research data of economics journals, not public-sector data."
292
292
  }
293
293
  ]);
294
- //# sourceMappingURL=portals-list.js.map
@@ -1,4 +1,5 @@
1
- import { type CkanClient } from "./client.js";
1
+ import { CkanClient } from "./client.js";
2
+ import { type EngineOptions } from "./engine.js";
2
3
  import type { Portal } from "./types.js";
3
4
  /**
4
5
  * One identity per portal, whatever form its URL was written in: host (without
@@ -34,4 +35,36 @@ export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
34
35
  export declare function withCheck(portal: Portal, check: PortalCheck, date: string): Portal;
35
36
  /** Run `fn` over `items` with at most `limit` calls in flight; results keep the input order. */
36
37
  export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
37
- //# sourceMappingURL=portals.d.ts.map
38
+ /** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
39
+ export declare const DEFAULT_CHECK_CONCURRENCY = 6;
40
+ /** Options for `checkPortalUrls` and `checkPortals`. */
41
+ export interface CheckPortalsOptions {
42
+ /** Client options for every check; `baseUrl` is replaced by each portal's URL. */
43
+ engineOptions?: EngineOptions;
44
+ /** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
45
+ createClient?: (options: EngineOptions) => CkanClient;
46
+ /** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
47
+ concurrency?: number;
48
+ /**
49
+ * When set, a failed check is repeated once after this pause (ms), so one
50
+ * timeout does not count as down. Unset (the default): a single try.
51
+ */
52
+ retryDelayMs?: number;
53
+ /** Injectable sleep for the retry pause, for tests. */
54
+ sleep?: (ms: number) => Promise<void>;
55
+ }
56
+ /**
57
+ * Check the CKAN portals at `urls` live (`checkPortal`), with at most
58
+ * `concurrency` checks in flight; results keep the input order. Never throws for
59
+ * one portal: a URL the client refuses becomes a failed check too. Rejects with
60
+ * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
61
+ */
62
+ export declare function checkPortalUrls(urls: readonly string[], options?: CheckPortalsOptions): Promise<PortalCheck[]>;
63
+ /**
64
+ * Check every portal in `portals` (the built-in list by default) live and fold
65
+ * each result into the entry (`withCheck`), dated `date` (default: today, UTC).
66
+ * What `ckan portals --check` prints.
67
+ */
68
+ export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
69
+ date?: string;
70
+ }): Promise<Portal[]>;
@@ -1,7 +1,10 @@
1
1
  // Known CKAN portals: lookup and a short live check. The list itself is in
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
- import { siteRoot } from "./client.js";
4
- import { CkanApiError, CkanNetworkError, CkanParseError } from "./errors.js";
3
+ import { CkanClient, siteRoot } from "./client.js";
4
+ import { validateBaseUrl } from "./engine.js";
5
+ import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
6
+ import { PORTALS } from "./portals-list.js";
7
+ import { assertValid, intRangeProblem } from "./validate.js";
5
8
  /**
6
9
  * One identity per portal, whatever form its URL was written in: host (without
7
10
  * `www.`) plus path, lower-cased, without scheme, trailing slash or a pasted
@@ -15,18 +18,29 @@ export function portalKey(url) {
15
18
  }
16
19
  /** A portal from `list` by its id (any case) or by any form of its URL. */
17
20
  export function findPortal(idOrUrl, list) {
21
+ if (typeof idOrUrl !== "string")
22
+ throw new CkanValidationError("Invalid portal: Expected a string.");
23
+ if (!Array.isArray(list))
24
+ throw new CkanValidationError("Invalid portal list: Expected an array.");
18
25
  const wanted = idOrUrl.trim().toLowerCase();
19
26
  const byId = list.find((p) => p.id.toLowerCase() === wanted);
20
27
  if (byId)
21
28
  return byId;
22
29
  let key;
23
30
  try {
24
- key = portalKey(idOrUrl);
31
+ key = portalKey(idOrUrl.trim());
25
32
  }
26
33
  catch {
27
34
  return undefined; // not a URL either
28
35
  }
29
- return list.find((p) => portalKey(p.url) === key);
36
+ return list.find((p) => {
37
+ try {
38
+ return portalKey(p.url) === key;
39
+ }
40
+ catch {
41
+ return false; // an entry without a usable url matches nothing
42
+ }
43
+ });
30
44
  }
31
45
  /**
32
46
  * A short live check of the CKAN portal a client points at. It works when `package_search?rows=0`
@@ -62,6 +76,17 @@ export async function checkPortal(client) {
62
76
  }
63
77
  return result;
64
78
  }
79
+ /**
80
+ * The first error `code` (`ENOTFOUND`, `ECONNREFUSED`, …) along a cause chain: the
81
+ * engine wraps a transport failure in a CkanNetworkError that names the request, so
82
+ * Node's code sits one or more levels down.
83
+ */
84
+ function errorCode(cause, depth = 0) {
85
+ if (typeof cause !== "object" || cause === null || depth > 5)
86
+ return undefined;
87
+ const code = cause.code;
88
+ return code !== undefined ? code : errorCode(cause.cause, depth + 1);
89
+ }
65
90
  /** A short, stable reason for a failed check. */
66
91
  function problemOf(err) {
67
92
  if (err instanceof CkanApiError) {
@@ -79,7 +104,7 @@ function problemOf(err) {
79
104
  return "invalid JSON";
80
105
  }
81
106
  if (err instanceof CkanNetworkError) {
82
- const code = err.cause?.code;
107
+ const code = errorCode(err.cause);
83
108
  if (code === "ENOTFOUND" || code === "EAI_AGAIN")
84
109
  return "host not found";
85
110
  if (code === "ECONNREFUSED")
@@ -90,6 +115,10 @@ function problemOf(err) {
90
115
  return "TLS certificate not verifiable";
91
116
  if (/timed out|deadline/.test(message))
92
117
  return "timeout";
118
+ // The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
119
+ const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
120
+ if (reason !== undefined)
121
+ return reason.slice(0, 80);
93
122
  }
94
123
  return message.slice(0, 80);
95
124
  }
@@ -121,4 +150,55 @@ export async function mapLimit(items, limit, fn) {
121
150
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
122
151
  return results;
123
152
  }
124
- //# sourceMappingURL=portals.js.map
153
+ /** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
154
+ export const DEFAULT_CHECK_CONCURRENCY = 6;
155
+ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
156
+ /**
157
+ * Check the CKAN portals at `urls` live (`checkPortal`), with at most
158
+ * `concurrency` checks in flight; results keep the input order. Never throws for
159
+ * one portal: a URL the client refuses becomes a failed check too. Rejects with
160
+ * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
161
+ */
162
+ export async function checkPortalUrls(urls, options = {}) {
163
+ if (!Array.isArray(urls))
164
+ throw new CkanValidationError("Invalid urls: Expected an array.");
165
+ const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
166
+ const { retryDelayMs } = options;
167
+ if (retryDelayMs !== undefined)
168
+ assertValid("retryDelayMs", retryDelayMs, intRangeProblem(0, Number.MAX_SAFE_INTEGER));
169
+ const createClient = options.createClient ?? ((engine) => new CkanClient(engine));
170
+ const sleep = options.sleep ?? realSleep;
171
+ return mapLimit(urls, concurrency, async (url) => {
172
+ let client;
173
+ try {
174
+ // Checked here, not left to the client: `baseUrl: undefined` selects the
175
+ // Hamburg default, so an entry without a url was checked against Hamburg and
176
+ // reported as working, with Hamburg's dataset count.
177
+ validateBaseUrl(url);
178
+ client = createClient({ ...options.engineOptions, baseUrl: url });
179
+ }
180
+ catch (err) {
181
+ return { ...failedCheck(), problem: problemOf(err) };
182
+ }
183
+ const first = await checkPortal(client);
184
+ if (first.working || retryDelayMs === undefined)
185
+ return first;
186
+ await sleep(retryDelayMs);
187
+ return checkPortal(client);
188
+ });
189
+ }
190
+ /**
191
+ * Check every portal in `portals` (the built-in list by default) live and fold
192
+ * each result into the entry (`withCheck`), dated `date` (default: today, UTC).
193
+ * What `ckan portals --check` prints.
194
+ */
195
+ export async function checkPortals(portals = PORTALS, options = {}) {
196
+ if (!Array.isArray(portals))
197
+ throw new CkanValidationError("Invalid portals: Expected an array.");
198
+ const date = options.date ?? new Date().toISOString().slice(0, 10);
199
+ const checks = await checkPortalUrls(portals.map((p) => p.url), options);
200
+ return portals.map((portal, i) => withCheck(portal, checks[i], date));
201
+ }
202
+ function failedCheck() {
203
+ return { working: false, problem: null, datasets: null, ckanVersion: null, siteTitle: null, siteUrl: null };
204
+ }
@@ -6,4 +6,3 @@ export type QueryParams = Record<string, QueryValue>;
6
6
  * Returns an empty string when no parameters survive filtering.
7
7
  */
8
8
  export declare function buildQueryString(params: QueryParams): string;
9
- //# sourceMappingURL=query.d.ts.map
@@ -30,4 +30,3 @@ export function buildQueryString(params) {
30
30
  // URLSearchParams encodes spaces as "+"; "%20" is more broadly interoperable.
31
31
  return search.toString().replace(/\+/g, "%20");
32
32
  }
33
- //# sourceMappingURL=query.js.map
@@ -101,4 +101,3 @@ export interface Portal {
101
101
  datasets: number | null;
102
102
  note: string | null;
103
103
  }
104
- //# sourceMappingURL=types.d.ts.map
@@ -5,4 +5,3 @@
5
5
  // nested and differ per portal (every instance adds its own extras), so they are
6
6
  // exposed as raw `JsonObject`s.
7
7
  export {};
8
- //# sourceMappingURL=types.js.map