@maschinenlesbar.org/ckan-cli 0.0.6 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/dist/src/cli/commands/catalogue.d.ts +1 -2
  2. package/dist/src/cli/commands/catalogue.js +11 -3
  3. package/dist/src/cli/commands/portal.d.ts +0 -1
  4. package/dist/src/cli/commands/portal.js +13 -12
  5. package/dist/src/cli/index.d.ts +0 -1
  6. package/dist/src/cli/index.js +0 -1
  7. package/dist/src/cli/io.d.ts +0 -1
  8. package/dist/src/cli/io.js +0 -1
  9. package/dist/src/cli/program.d.ts +0 -1
  10. package/dist/src/cli/program.js +2 -3
  11. package/dist/src/cli/run.d.ts +0 -1
  12. package/dist/src/cli/run.js +9 -2
  13. package/dist/src/cli/shared.d.ts +10 -15
  14. package/dist/src/cli/shared.js +29 -54
  15. package/dist/src/client/client.d.ts +7 -6
  16. package/dist/src/client/client.js +38 -22
  17. package/dist/src/client/engine.d.ts +28 -6
  18. package/dist/src/client/engine.js +40 -30
  19. package/dist/src/client/errors.d.ts +7 -1
  20. package/dist/src/client/errors.js +7 -1
  21. package/dist/src/client/http.d.ts +0 -1
  22. package/dist/src/client/http.js +41 -33
  23. package/dist/src/client/index.d.ts +6 -5
  24. package/dist/src/client/index.js +4 -4
  25. package/dist/src/client/portals-list.d.ts +0 -1
  26. package/dist/src/client/portals-list.js +0 -1
  27. package/dist/src/client/portals.d.ts +35 -2
  28. package/dist/src/client/portals.js +48 -3
  29. package/dist/src/client/query.d.ts +0 -1
  30. package/dist/src/client/query.js +0 -1
  31. package/dist/src/client/types.d.ts +0 -1
  32. package/dist/src/client/types.js +0 -1
  33. package/dist/src/client/validate.d.ts +55 -0
  34. package/dist/src/client/validate.js +119 -0
  35. package/dist/src/index.d.ts +0 -1
  36. package/dist/src/index.js +0 -1
  37. package/package.json +2 -1
  38. package/dist/src/cli/commands/catalogue.d.ts.map +0 -1
  39. package/dist/src/cli/commands/catalogue.js.map +0 -1
  40. package/dist/src/cli/commands/portal.d.ts.map +0 -1
  41. package/dist/src/cli/commands/portal.js.map +0 -1
  42. package/dist/src/cli/index.d.ts.map +0 -1
  43. package/dist/src/cli/index.js.map +0 -1
  44. package/dist/src/cli/io.d.ts.map +0 -1
  45. package/dist/src/cli/io.js.map +0 -1
  46. package/dist/src/cli/program.d.ts.map +0 -1
  47. package/dist/src/cli/program.js.map +0 -1
  48. package/dist/src/cli/run.d.ts.map +0 -1
  49. package/dist/src/cli/run.js.map +0 -1
  50. package/dist/src/cli/shared.d.ts.map +0 -1
  51. package/dist/src/cli/shared.js.map +0 -1
  52. package/dist/src/client/client.d.ts.map +0 -1
  53. package/dist/src/client/client.js.map +0 -1
  54. package/dist/src/client/engine.d.ts.map +0 -1
  55. package/dist/src/client/engine.js.map +0 -1
  56. package/dist/src/client/errors.d.ts.map +0 -1
  57. package/dist/src/client/errors.js.map +0 -1
  58. package/dist/src/client/http.d.ts.map +0 -1
  59. package/dist/src/client/http.js.map +0 -1
  60. package/dist/src/client/index.d.ts.map +0 -1
  61. package/dist/src/client/index.js.map +0 -1
  62. package/dist/src/client/portals-list.d.ts.map +0 -1
  63. package/dist/src/client/portals-list.js.map +0 -1
  64. package/dist/src/client/portals.d.ts.map +0 -1
  65. package/dist/src/client/portals.js.map +0 -1
  66. package/dist/src/client/query.d.ts.map +0 -1
  67. package/dist/src/client/query.js.map +0 -1
  68. package/dist/src/client/types.d.ts.map +0 -1
  69. package/dist/src/client/types.js.map +0 -1
  70. package/dist/src/index.d.ts.map +0 -1
  71. package/dist/src/index.js.map +0 -1
@@ -1,14 +1,28 @@
1
1
  // The request engine: turns logical (method, path, query) calls into HTTP
2
2
  // requests via a Transport, applies retry/backoff for transient statuses
3
3
  // (429, 503), and decodes responses.
4
- import { nodeHttpTransport } from "./http.js";
4
+ import { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  import { buildQueryString } from "./query.js";
6
- import { CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
6
+ import { CkanApiError, CkanParseError, describeCkanError, redactUrl } from "./errors.js";
7
+ import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
7
8
  export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
8
9
  const DEFAULT_USER_AGENT = "ckan-cli";
9
10
  const DEFAULT_MAX_RESPONSE_BYTES = 100 * 1024 * 1024;
10
- /** Most automatic retries the CLI's --max-retries accepts. */
11
+ /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
11
12
  export const MAX_RETRIES = 10;
13
+ /** Most redirects the engine follows (`maxRedirects`). */
14
+ export const MAX_REDIRECTS = 10;
15
+ /**
16
+ * Check a value for an HTTP header (headerValueProblem) and return it, or throw
17
+ * CkanValidationError `Invalid <name>: …`.
18
+ */
19
+ export function assertHeaderValue(name, value) {
20
+ return assertValid(name, value, headerValueProblem);
21
+ }
22
+ /** Check an optional numeric engine option against `[min, max]` (CkanValidationError). */
23
+ function intOption(name, value, min, max) {
24
+ return value === undefined ? undefined : assertValid(name, value, intRangeProblem(min, max));
25
+ }
12
26
  /**
13
27
  * Longest `Retry-After` the engine waits out before retrying a 429/503. When the
14
28
  * server asks for longer, the engine does not retry at all and surfaces the error at
@@ -83,22 +97,15 @@ export function sanitizeServerText(text) {
83
97
  return out.replace(/\s+/g, " ").trim();
84
98
  }
85
99
  /**
86
- * Reject a base URL whose scheme is not http(s). The default transport already
87
- * gates this per hop, but the engine is exported as a library and may be handed a
88
- * custom transport that does no such check, so gate the configured base URL here
89
- * too (a `file:`/`ftp:` base URL fails fast with a typed error).
100
+ * Check a base URL (baseUrlProblem: not blank, no whitespace or control
101
+ * characters, an absolute http(s) URL, no query string or fragment) and return it
102
+ * without trailing slashes. Throws CkanValidationError `Invalid base URL: …` — a
103
+ * configuration mistake, not a CkanNetworkError. The RequestEngine and CkanClient
104
+ * constructors call it, so a custom transport never sees a bad base URL; the
105
+ * default transport still re-checks the scheme on every hop.
90
106
  */
91
- function assertHttpScheme(baseUrl) {
92
- let url;
93
- try {
94
- url = new URL(baseUrl);
95
- }
96
- catch {
97
- throw new CkanNetworkError(`Invalid base URL: ${redactUrl(baseUrl)}`);
98
- }
99
- if (url.protocol !== "http:" && url.protocol !== "https:") {
100
- throw new CkanNetworkError(`Unsupported protocol "${url.protocol}" in base URL: ${redactUrl(baseUrl)}`);
101
- }
107
+ export function validateBaseUrl(raw) {
108
+ return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
102
109
  }
103
110
  // The headers the engine sets itself, under the exact keys it uses. They are the
104
111
  // only ones that follow a cross-origin redirect.
@@ -124,18 +131,22 @@ export class RequestEngine {
124
131
  maxResponseBytes;
125
132
  sleep;
126
133
  constructor(options = {}) {
127
- this.baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
128
- // Re-check the base-URL scheme here, not only in the default transport: a
129
- // library consumer that injects a custom transport would otherwise get no
130
- // gating at all, and could be steered to a non-http(s) scheme.
131
- assertHttpScheme(this.baseUrl);
134
+ // The raw value, checked before the slash strip (the engine glues it into
135
+ // every URL), and here rather than only in the default transport: a library
136
+ // consumer that injects a custom transport would otherwise get no gating at
137
+ // all, and could be steered to a non-http(s) scheme. Only undefined selects
138
+ // the default.
139
+ this.baseUrl = validateBaseUrl(options.baseUrl ?? DEFAULT_BASE_URL);
132
140
  this.transport = options.transport ?? nodeHttpTransport;
133
- this.userAgent = options.userAgent ?? DEFAULT_USER_AGENT;
134
- this.timeoutMs = options.timeoutMs ?? 30_000;
135
- this.maxRetries = options.maxRetries ?? 2;
136
- this.retryDelayMs = options.retryDelayMs ?? 200;
137
- this.maxRedirects = options.maxRedirects ?? 5;
138
- this.maxResponseBytes = options.maxResponseBytes ?? DEFAULT_MAX_RESPONSE_BYTES;
141
+ // Only undefined selects the default; a blank or unsendable value is refused.
142
+ this.userAgent =
143
+ options.userAgent === undefined ? DEFAULT_USER_AGENT : assertHeaderValue("userAgent", options.userAgent);
144
+ this.timeoutMs = intOption("timeoutMs", options.timeoutMs, 0, MAX_TIMEOUT_MS) ?? 30_000;
145
+ this.maxRetries = intOption("maxRetries", options.maxRetries, 0, MAX_RETRIES) ?? 2;
146
+ this.retryDelayMs = intOption("retryDelayMs", options.retryDelayMs, 0, Number.MAX_SAFE_INTEGER) ?? 200;
147
+ this.maxRedirects = intOption("maxRedirects", options.maxRedirects, 0, MAX_REDIRECTS) ?? 5;
148
+ this.maxResponseBytes =
149
+ intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
139
150
  this.sleep = options.sleep ?? realSleep;
140
151
  }
141
152
  /** Build a fully-qualified URL from a path and optional query parameters. */
@@ -294,4 +305,3 @@ function resolveLocation(location, base) {
294
305
  return undefined;
295
306
  }
296
307
  }
297
- //# sourceMappingURL=engine.js.map
@@ -36,6 +36,13 @@ export declare class CkanNetworkError extends CkanError {
36
36
  /** The response body could not be parsed as the expected JSON shape. */
37
37
  export declare class CkanParseError extends CkanError {
38
38
  }
39
+ /**
40
+ * An input the library refuses before sending any request: a client option or a
41
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
42
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
43
+ */
44
+ export declare class CkanValidationError extends CkanError {
45
+ }
39
46
  /**
40
47
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
41
48
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -43,4 +50,3 @@ export declare class CkanParseError extends CkanError {
43
50
  * (`{"__type": "Validation Error", "rows": ["Invalid integer"]}`).
44
51
  */
45
52
  export declare function describeCkanError(error: unknown): string;
46
- //# sourceMappingURL=errors.d.ts.map
@@ -58,6 +58,13 @@ export class CkanNetworkError extends CkanError {
58
58
  /** The response body could not be parsed as the expected JSON shape. */
59
59
  export class CkanParseError extends CkanError {
60
60
  }
61
+ /**
62
+ * An input the library refuses before sending any request: a client option or a
63
+ * method parameter that breaks one of the rules in `validate.ts`. The message is
64
+ * `Invalid <name>: <reason>`. The CLI reports it as a usage error.
65
+ */
66
+ export class CkanValidationError extends CkanError {
67
+ }
61
68
  /**
62
69
  * Turn a CKAN `error` object into one readable line. CKAN has two shapes: a
63
70
  * `message` (`{"__type": "Not Found Error", "message": "Not found"}`), and a
@@ -91,4 +98,3 @@ function solrReason(message) {
91
98
  return undefined;
92
99
  return match[1].replace(/\\+(['"])/g, "$1").trim();
93
100
  }
94
- //# sourceMappingURL=errors.js.map
@@ -28,4 +28,3 @@ export declare const MAX_TIMEOUT_MS = 2147483647;
28
28
  * (connection errors, timeouts, malformed URLs).
29
29
  */
30
30
  export declare const nodeHttpTransport: Transport;
31
- //# sourceMappingURL=http.d.ts.map
@@ -50,40 +50,49 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
50
50
  clearTimeout(deadline);
51
51
  reject(err);
52
52
  };
53
- const req = driver.request(url, {
54
- method: request.method,
55
- headers: request.headers,
56
- }, (res) => {
57
- const chunks = [];
58
- let received = 0;
59
- let aborted = false;
60
- res.on("data", (chunk) => {
61
- if (aborted)
62
- return;
63
- received += chunk.length;
64
- if (maxBytes !== undefined && received > maxBytes) {
65
- aborted = true;
66
- res.destroy();
67
- settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
68
- return;
69
- }
70
- chunks.push(chunk);
71
- });
72
- res.on("end", () => {
73
- if (aborted)
74
- return;
75
- settleResolve({
76
- status: res.statusCode ?? 0,
77
- headers: res.headers,
78
- body: Buffer.concat(chunks),
53
+ // Node validates the headers synchronously and throws a bare TypeError
54
+ // (ERR_INVALID_CHAR) for one it cannot send; report it as a typed failure.
55
+ let req;
56
+ try {
57
+ req = driver.request(url, {
58
+ method: request.method,
59
+ headers: request.headers,
60
+ }, (res) => {
61
+ const chunks = [];
62
+ let received = 0;
63
+ let aborted = false;
64
+ res.on("data", (chunk) => {
65
+ if (aborted)
66
+ return;
67
+ received += chunk.length;
68
+ if (maxBytes !== undefined && received > maxBytes) {
69
+ aborted = true;
70
+ res.destroy();
71
+ settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
72
+ return;
73
+ }
74
+ chunks.push(chunk);
75
+ });
76
+ res.on("end", () => {
77
+ if (aborted)
78
+ return;
79
+ settleResolve({
80
+ status: res.statusCode ?? 0,
81
+ headers: res.headers,
82
+ body: Buffer.concat(chunks),
83
+ });
84
+ });
85
+ res.on("error", (err) => {
86
+ if (aborted)
87
+ return; // we already rejected with the size-cap error
88
+ settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
79
89
  });
80
90
  });
81
- res.on("error", (err) => {
82
- if (aborted)
83
- return; // we already rejected with the size-cap error
84
- settleReject(new CkanNetworkError(`Response stream error: ${err.message}`, { cause: err }));
85
- });
86
- });
91
+ }
92
+ catch (err) {
93
+ reject(new CkanNetworkError(`Invalid request: ${err instanceof Error ? err.message : String(err)}`, { cause: err }));
94
+ return;
95
+ }
87
96
  if (request.timeoutMs && request.timeoutMs > 0) {
88
97
  // Two complementary guards, both using timeoutMs:
89
98
  // - setTimeout: a socket-*inactivity* timeout (no bytes for timeoutMs).
@@ -108,4 +117,3 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
108
117
  req.write(request.body);
109
118
  req.end();
110
119
  });
111
- //# sourceMappingURL=http.js.map
@@ -1,13 +1,14 @@
1
1
  export { CkanClient, siteRoot } from "./client.js";
2
- export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
2
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
9
+ export { assertValid } from "./validate.js";
10
+ export type { Problem } from "./validate.js";
9
11
  export { PORTALS } from "./portals-list.js";
10
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
11
- export type { PortalCheck } from "./portals.js";
12
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
13
+ export type { CheckPortalsOptions, PortalCheck } from "./portals.js";
12
14
  export * from "./types.js";
13
- //# sourceMappingURL=index.d.ts.map
@@ -1,10 +1,10 @@
1
1
  // Public entry point for the API client library.
2
2
  export { CkanClient, siteRoot } from "./client.js";
3
- export { RequestEngine, DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, } from "./engine.js";
3
+ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, describeCkanError, redactUrl, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
7
+ export { assertValid } from "./validate.js";
7
8
  export { PORTALS } from "./portals-list.js";
8
- export { checkPortal, findPortal, mapLimit, portalKey, withCheck } from "./portals.js";
9
+ export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
9
10
  export * from "./types.js";
10
- //# sourceMappingURL=index.js.map
@@ -1,3 +1,2 @@
1
1
  import type { Portal } from "./types.js";
2
2
  export declare const PORTALS: readonly Portal[];
3
- //# sourceMappingURL=portals-list.d.ts.map
@@ -291,4 +291,3 @@ export const PORTALS = frozen([
291
291
  "note": "Research data of economics journals, not public-sector data."
292
292
  }
293
293
  ]);
294
- //# sourceMappingURL=portals-list.js.map
@@ -1,4 +1,5 @@
1
- import { type CkanClient } from "./client.js";
1
+ import { CkanClient } from "./client.js";
2
+ import type { EngineOptions } from "./engine.js";
2
3
  import type { Portal } from "./types.js";
3
4
  /**
4
5
  * One identity per portal, whatever form its URL was written in: host (without
@@ -34,4 +35,36 @@ export declare function checkPortal(client: CkanClient): Promise<PortalCheck>;
34
35
  export declare function withCheck(portal: Portal, check: PortalCheck, date: string): Portal;
35
36
  /** Run `fn` over `items` with at most `limit` calls in flight; results keep the input order. */
36
37
  export declare function mapLimit<T, R>(items: readonly T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
37
- //# sourceMappingURL=portals.d.ts.map
38
+ /** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
39
+ export declare const DEFAULT_CHECK_CONCURRENCY = 6;
40
+ /** Options for `checkPortalUrls` and `checkPortals`. */
41
+ export interface CheckPortalsOptions {
42
+ /** Client options for every check; `baseUrl` is replaced by each portal's URL. */
43
+ engineOptions?: EngineOptions;
44
+ /** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
45
+ createClient?: (options: EngineOptions) => CkanClient;
46
+ /** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
47
+ concurrency?: number;
48
+ /**
49
+ * When set, a failed check is repeated once after this pause (ms), so one
50
+ * timeout does not count as down. Unset (the default): a single try.
51
+ */
52
+ retryDelayMs?: number;
53
+ /** Injectable sleep for the retry pause, for tests. */
54
+ sleep?: (ms: number) => Promise<void>;
55
+ }
56
+ /**
57
+ * Check the CKAN portals at `urls` live (`checkPortal`), with at most
58
+ * `concurrency` checks in flight; results keep the input order. Never throws for
59
+ * one portal: a URL the client refuses becomes a failed check too. Rejects with
60
+ * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
61
+ */
62
+ export declare function checkPortalUrls(urls: readonly string[], options?: CheckPortalsOptions): Promise<PortalCheck[]>;
63
+ /**
64
+ * Check every portal in `portals` (the built-in list by default) live and fold
65
+ * each result into the entry (`withCheck`), dated `date` (default: today, UTC).
66
+ * What `ckan portals --check` prints.
67
+ */
68
+ export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
69
+ date?: string;
70
+ }): Promise<Portal[]>;
@@ -1,7 +1,9 @@
1
1
  // Known CKAN portals: lookup and a short live check. The list itself is in
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
- import { siteRoot } from "./client.js";
3
+ import { CkanClient, siteRoot } from "./client.js";
4
4
  import { CkanApiError, CkanNetworkError, CkanParseError } from "./errors.js";
5
+ import { PORTALS } from "./portals-list.js";
6
+ import { assertValid, intRangeProblem } from "./validate.js";
5
7
  /**
6
8
  * One identity per portal, whatever form its URL was written in: host (without
7
9
  * `www.`) plus path, lower-cased, without scheme, trailing slash or a pasted
@@ -21,7 +23,7 @@ export function findPortal(idOrUrl, list) {
21
23
  return byId;
22
24
  let key;
23
25
  try {
24
- key = portalKey(idOrUrl);
26
+ key = portalKey(idOrUrl.trim());
25
27
  }
26
28
  catch {
27
29
  return undefined; // not a URL either
@@ -121,4 +123,47 @@ export async function mapLimit(items, limit, fn) {
121
123
  await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
122
124
  return results;
123
125
  }
124
- //# sourceMappingURL=portals.js.map
126
+ /** How many portals `checkPortals` / `checkPortalUrls` check at once by default. */
127
+ export const DEFAULT_CHECK_CONCURRENCY = 6;
128
+ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
129
+ /**
130
+ * Check the CKAN portals at `urls` live (`checkPortal`), with at most
131
+ * `concurrency` checks in flight; results keep the input order. Never throws for
132
+ * one portal: a URL the client refuses becomes a failed check too. Rejects with
133
+ * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
134
+ */
135
+ export async function checkPortalUrls(urls, options = {}) {
136
+ const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
137
+ const { retryDelayMs } = options;
138
+ if (retryDelayMs !== undefined)
139
+ assertValid("retryDelayMs", retryDelayMs, intRangeProblem(0, Number.MAX_SAFE_INTEGER));
140
+ const createClient = options.createClient ?? ((engine) => new CkanClient(engine));
141
+ const sleep = options.sleep ?? realSleep;
142
+ return mapLimit(urls, concurrency, async (url) => {
143
+ let client;
144
+ try {
145
+ client = createClient({ ...options.engineOptions, baseUrl: url });
146
+ }
147
+ catch (err) {
148
+ return { ...failedCheck(), problem: problemOf(err) };
149
+ }
150
+ const first = await checkPortal(client);
151
+ if (first.working || retryDelayMs === undefined)
152
+ return first;
153
+ await sleep(retryDelayMs);
154
+ return checkPortal(client);
155
+ });
156
+ }
157
+ /**
158
+ * Check every portal in `portals` (the built-in list by default) live and fold
159
+ * each result into the entry (`withCheck`), dated `date` (default: today, UTC).
160
+ * What `ckan portals --check` prints.
161
+ */
162
+ export async function checkPortals(portals = PORTALS, options = {}) {
163
+ const date = options.date ?? new Date().toISOString().slice(0, 10);
164
+ const checks = await checkPortalUrls(portals.map((p) => p.url), options);
165
+ return portals.map((portal, i) => withCheck(portal, checks[i], date));
166
+ }
167
+ function failedCheck() {
168
+ return { working: false, problem: null, datasets: null, ckanVersion: null, siteTitle: null, siteUrl: null };
169
+ }
@@ -6,4 +6,3 @@ export type QueryParams = Record<string, QueryValue>;
6
6
  * Returns an empty string when no parameters survive filtering.
7
7
  */
8
8
  export declare function buildQueryString(params: QueryParams): string;
9
- //# sourceMappingURL=query.d.ts.map
@@ -30,4 +30,3 @@ export function buildQueryString(params) {
30
30
  // URLSearchParams encodes spaces as "+"; "%20" is more broadly interoperable.
31
31
  return search.toString().replace(/\+/g, "%20");
32
32
  }
33
- //# sourceMappingURL=query.js.map
@@ -101,4 +101,3 @@ export interface Portal {
101
101
  datasets: number | null;
102
102
  note: string | null;
103
103
  }
104
- //# sourceMappingURL=types.d.ts.map
@@ -5,4 +5,3 @@
5
5
  // nested and differ per portal (every instance adds its own extras), so they are
6
6
  // exposed as raw `JsonObject`s.
7
7
  export {};
8
- //# sourceMappingURL=types.js.map
@@ -0,0 +1,55 @@
1
+ /** A rule: the reason `value` is invalid (one sentence), or `undefined` when it is valid. */
2
+ export type Problem<T = unknown> = (value: T) => string | undefined;
3
+ /**
4
+ * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
5
+ * something wrong with `value`; otherwise return `value` unchanged. A method that
6
+ * returns a promise calls this inside its async body, so a bad input rejects
7
+ * rather than throwing synchronously.
8
+ */
9
+ export declare function assertValid<T>(name: string, value: T, problem: Problem<T>): T;
10
+ /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
11
+ export declare function isBlank(value: string): boolean;
12
+ /**
13
+ * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
14
+ * no filter, so a blank search, filter, sort, facet field or tag query would
15
+ * silently return the unfiltered result. Values that are not strings pass.
16
+ */
17
+ export declare function blankProblem(value: unknown): string | undefined;
18
+ /**
19
+ * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
20
+ * A negative, fractional or non-finite value would go upstream as given, and
21
+ * the `all_fields` pager would step on from it.
22
+ */
23
+ export declare function countProblem(value: number): string | undefined;
24
+ /** A `facet_limit`: -1 (every value) or a non-negative safe integer. */
25
+ export declare function facetLimitProblem(value: number): string | undefined;
26
+ /**
27
+ * A rule for a safe integer within `[min, max]`; the messages are the CLI's
28
+ * (`Must be >= 0.`), so a flag and a client option report the same reason.
29
+ */
30
+ export declare function intRangeProblem(min: number, max: number): Problem<number>;
31
+ /**
32
+ * A value for an HTTP header (the User-Agent): not blank, no C0 control other
33
+ * than tab, no DEL, nothing above U+00FF. Node's HTTP layer refuses those at
34
+ * request time with an untyped "Invalid character in header content", and a
35
+ * custom transport might send a CR/LF on as a forged header. Checked by char code
36
+ * so the source stays free of control bytes.
37
+ */
38
+ export declare function headerValueProblem(value: string): string | undefined;
39
+ /**
40
+ * A base URL (the CKAN site URL), checked in this order:
41
+ *
42
+ * - not blank;
43
+ * - no surrounding or embedded whitespace and no control characters: `new URL`
44
+ * trims and drops tab/newline silently, but the client glues the value into
45
+ * every request URL as given, so a trailing space would request another path
46
+ * (`/ckan%20/api/3/...`) or reach a custom transport raw;
47
+ * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
48
+ * transport);
49
+ * - no query string or fragment: the API path is appended, so it would land in
50
+ * front of `/api/3/action`.
51
+ *
52
+ * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
53
+ * reasons never quote the value, so a credential in it cannot leak through them.
54
+ */
55
+ export declare function baseUrlProblem(value: string): string | undefined;
@@ -0,0 +1,119 @@
1
+ // Input rules of the library, as pure functions. Each `…Problem` function returns
2
+ // the reason a value is invalid, or `undefined` when it is valid. Client methods
3
+ // enforce them with `assertValid` before any request; the CLI's value parsers call
4
+ // the same functions, so a rule is written once and the CLI and the library agree.
5
+ import { CkanValidationError } from "./errors.js";
6
+ /**
7
+ * Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
8
+ * something wrong with `value`; otherwise return `value` unchanged. A method that
9
+ * returns a promise calls this inside its async body, so a bad input rejects
10
+ * rather than throwing synchronously.
11
+ */
12
+ export function assertValid(name, value, problem) {
13
+ const reason = problem(value);
14
+ if (reason !== undefined)
15
+ throw new CkanValidationError(`Invalid ${name}: ${reason}`);
16
+ return value;
17
+ }
18
+ /** True for an empty or whitespace-only string: CKAN reads it as "not given". */
19
+ export function isBlank(value) {
20
+ return value.trim() === "";
21
+ }
22
+ /**
23
+ * A blank string, or a list with a blank entry. CKAN treats an empty parameter as
24
+ * no filter, so a blank search, filter, sort, facet field or tag query would
25
+ * silently return the unfiltered result. Values that are not strings pass.
26
+ */
27
+ export function blankProblem(value) {
28
+ const blank = (v) => typeof v === "string" && isBlank(v);
29
+ if (blank(value) || (Array.isArray(value) && value.some(blank)))
30
+ return "Expected a non-empty value.";
31
+ return undefined;
32
+ }
33
+ /**
34
+ * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
35
+ * A negative, fractional or non-finite value would go upstream as given, and
36
+ * the `all_fields` pager would step on from it.
37
+ */
38
+ export function countProblem(value) {
39
+ return Number.isSafeInteger(value) && value >= 0 ? undefined : "Expected a non-negative integer.";
40
+ }
41
+ /** A `facet_limit`: -1 (every value) or a non-negative safe integer. */
42
+ export function facetLimitProblem(value) {
43
+ return value === -1 || countProblem(value) === undefined ? undefined : "Expected -1 or a non-negative integer.";
44
+ }
45
+ /**
46
+ * A rule for a safe integer within `[min, max]`; the messages are the CLI's
47
+ * (`Must be >= 0.`), so a flag and a client option report the same reason.
48
+ */
49
+ export function intRangeProblem(min, max) {
50
+ return (value) => {
51
+ if (!Number.isSafeInteger(value))
52
+ return `Expected an integer from ${min} to ${max}.`;
53
+ if (value < min)
54
+ return `Must be >= ${min}.`;
55
+ if (value > max)
56
+ return `Must be <= ${max}.`;
57
+ return undefined;
58
+ };
59
+ }
60
+ /**
61
+ * A value for an HTTP header (the User-Agent): not blank, no C0 control other
62
+ * than tab, no DEL, nothing above U+00FF. Node's HTTP layer refuses those at
63
+ * request time with an untyped "Invalid character in header content", and a
64
+ * custom transport might send a CR/LF on as a forged header. Checked by char code
65
+ * so the source stays free of control bytes.
66
+ */
67
+ export function headerValueProblem(value) {
68
+ const blank = blankProblem(value);
69
+ if (blank !== undefined)
70
+ return blank;
71
+ for (let i = 0; i < value.length; i++) {
72
+ const c = value.charCodeAt(i);
73
+ if ((c < 0x20 && c !== 0x09) || c === 0x7f)
74
+ return "Value contains control characters.";
75
+ if (c > 0xff)
76
+ return "Value contains characters outside Latin-1 (above U+00FF).";
77
+ }
78
+ return undefined;
79
+ }
80
+ /**
81
+ * A base URL (the CKAN site URL), checked in this order:
82
+ *
83
+ * - not blank;
84
+ * - no surrounding or embedded whitespace and no control characters: `new URL`
85
+ * trims and drops tab/newline silently, but the client glues the value into
86
+ * every request URL as given, so a trailing space would request another path
87
+ * (`/ckan%20/api/3/...`) or reach a custom transport raw;
88
+ * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
89
+ * transport);
90
+ * - no query string or fragment: the API path is appended, so it would land in
91
+ * front of `/api/3/action`.
92
+ *
93
+ * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
94
+ * reasons never quote the value, so a credential in it cannot leak through them.
95
+ */
96
+ export function baseUrlProblem(value) {
97
+ if (isBlank(value))
98
+ return "Expected an absolute http(s) URL.";
99
+ if (value !== value.trim())
100
+ return "A base URL cannot have surrounding whitespace.";
101
+ for (const ch of value) {
102
+ const c = ch.codePointAt(0) ?? 0;
103
+ if (c < 0x20 || c === 0x7f || /\s/u.test(ch))
104
+ return "A base URL cannot contain whitespace or control characters.";
105
+ }
106
+ let url;
107
+ try {
108
+ url = new URL(value);
109
+ }
110
+ catch {
111
+ return "Expected an absolute http(s) URL.";
112
+ }
113
+ if (url.protocol !== "http:" && url.protocol !== "https:") {
114
+ return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
115
+ }
116
+ if (/[?#]/.test(value))
117
+ return "Expected a site URL without a query string or fragment.";
118
+ return undefined;
119
+ }
@@ -1,2 +1 @@
1
1
  export * from "./client/index.js";
2
- //# sourceMappingURL=index.d.ts.map
package/dist/src/index.js CHANGED
@@ -1,3 +1,2 @@
1
1
  // Library root. Import from here when using ckan-cli as a dependency.
2
2
  export * from "./client/index.js";
3
- //# sourceMappingURL=index.js.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.0.6",
3
+ "version": "0.1.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {
@@ -16,6 +16,7 @@
16
16
  },
17
17
  "files": [
18
18
  "dist/src",
19
+ "!dist/src/**/*.map",
19
20
  "LICENSING.md",
20
21
  "CONTRIBUTING.md"
21
22
  ],
@@ -1 +0,0 @@
1
- {"version":3,"file":"catalogue.d.ts","sourceRoot":"","sources":["../../../../src/cli/commands/catalogue.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACzC,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,UAAU,CAAC;AAsBxC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,GAAG,IAAI,CAwI/E"}
@@ -1 +0,0 @@
1
- {"version":3,"file":"catalogue.js","sourceRoot":"","sources":["../../../../src/cli/commands/catalogue.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,YAAY;AAIZ,OAAO,EACL,MAAM,EACN,eAAe,EACf,eAAe,EACf,WAAW,EACX,aAAa,EACb,gBAAgB,EAChB,UAAU,GACX,MAAM,cAAc,CAAC;AAEtB;;;GAGG;AACH,MAAM,UAAU,GAAG,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;AAE/D,gFAAgF;AAChF,SAAS,eAAe,CAAC,KAAa;IACpC,OAAO,KAAK,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;AAClD,CAAC;AAED,MAAM,UAAU,yBAAyB,CAAC,OAAgB,EAAE,IAAa;IACvE,OAAO;SACJ,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,qCAAqC,CAAC;SAClD,QAAQ,CAAC,SAAS,EAAE,8CAA8C,EAAE,aAAa,CAAC;SAClF,MAAM,CAAC,YAAY,EAAE,iDAAiD,EAAE,WAAW,CAAC;SACpF,MAAM,CAAC,aAAa,EAAE,mBAAmB,EAAE,WAAW,CAAC;SACvD,MAAM,CAAC,eAAe,EAAE,+BAA+B,EAAE,gBAAgB,CAAC;SAC1E,MAAM,CAAC,eAAe,EAAE,qEAAqE,EAAE,eAAe,CAAC;SAC/G,MAAM,CAAC,iBAAiB,EAAE,uDAAuD,EAAE,eAAe,CAAC;SACnG,MAAM,CAAC,mBAAmB,EAAE,6CAA6C,EAAE,eAAe,CAAC;SAC3F,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE;QACvD,UAAU,CACR,IAAI,EACJ,MAAM,EACN,MAAM,MAAM,CAAC,aAAa,CAAC;YACzB,CAAC,EAAE,KAAK;YACR,IAAI,EAAE,IAAI,CAAC,MAAM,CAAuB;YACxC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAuB;YAC1C,IAAI,EAAE,IAAI,CAAC,MAAM,CAAuB;YACxC,EAAE,EAAE,IAAI,CAAC,IAAI,CAAyB;YACtC,WAAW,EAAE,IAAI,CAAC,OAAO,CAAyB;YAClD,WAAW,EAAE,IAAI,CAAC,YAAY,CAAuB;SACtD,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,SAAS,CAAC;SAClB,WAAW,CAAC,gCAAgC,CAAC;SAC7C,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,CAAC;SAC7C,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,WAAW,CAAC,EAAG,CAAC,CAAC,CAAC;IAC1D,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,wCAAwC,CAAC;SACrD,QAAQ,CAAC,MAAM,EAAE,aAAa,EAAE,aAAa,CAAC;SAC9C,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,YAAY,CAAC,EAAG,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,cAAc,CAAC;SACvB,WAAW,CAAC,iDAAiD,CAAC;SAC9D,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,CAAC;SAC7C,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,gBAAgB,CAAC,EAAG,CAAC,CAAC,CAAC;IAC/D,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,OAAO,CAAC;SAChB,WAAW,CAAC,+CAA+C,CAAC;SAC5D,QAAQ,CAAC,MAAM,EAAE,YAAY,EAAE,aAAa,CAAC;SAC7C,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC,EAAE,CAAC,EAAE,EAAE;QAC9C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,SAAS,CAAC,EAAG,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,UAAU,CAAC;SACnB,WAAW,CAAC,oBAAoB,CAAC;SACjC,MAAM,CAAC,aAAa,EAAE,qCAAqC,EAAE,UAAU,CAAC;SACxE,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE;QAC9C,UAAU,CACR,IAAI,EACJ,MAAM,EACN,MAAM,MAAM,CAAC,WAAW,CAAC;YACvB,KAAK,EAAE,IAAI,CAAC,OAAO,CAAuB;YAC1C,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAuB;SAC7C,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,eAAe,CAAC;SACxB,WAAW,CAAC,sCAAsC,CAAC;SACnD,MAAM,CAAC,cAAc,EAAE,sCAAsC,CAAC;SAC9D,MAAM,CAAC,aAAa,EAAE,uCAAuC,EAAE,UAAU,CAAC;SAC1E,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE;QAC9C,UAAU,CACR,IAAI,EACJ,MAAM,EACN,MAAM,MAAM,CAAC,gBAAgB,CAAC;YAC5B,UAAU,EAAE,IAAI,CAAC,WAAW,CAAwB;YACpD,KAAK,EAAE,IAAI,CAAC,OAAO,CAAuB;YAC1C,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAuB;SAC7C,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,QAAQ,CAAC;SACjB,WAAW,CAAC,iCAAiC,CAAC;SAC9C,MAAM,CAAC,cAAc,EAAE,sCAAsC,CAAC;SAC9D,MAAM,CAAC,aAAa,EAAE,uCAAuC,EAAE,UAAU,CAAC;SAC1E,MAAM,CAAC,cAAc,EAAE,mBAAmB,EAAE,WAAW,CAAC;SACxD,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE;QAC9C,UAAU,CACR,IAAI,EACJ,MAAM,EACN,MAAM,MAAM,CAAC,SAAS,CAAC;YACrB,UAAU,EAAE,IAAI,CAAC,WAAW,CAAwB;YACpD,KAAK,EAAE,IAAI,CAAC,OAAO,CAAuB;YAC1C,MAAM,EAAE,IAAI,CAAC,QAAQ,CAAuB;SAC7C,CAAC,CACH,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IAEJ,OAAO;SACJ,OAAO,CAAC,MAAM,CAAC;SACf,WAAW,CAAC,WAAW,CAAC;SACxB,MAAM,CAAC,qBAAqB,EAAE,qCAAqC,EAAE,gBAAgB,CAAC;SACtF,MAAM,CACL,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE;QAC9C,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,OAAO,CAAuB,EAAE,CAAC,CAAC,CAAC;IACjG,CAAC,CAAC,CACH,CAAC;AACN,CAAC"}