@maschinenlesbar.org/ckan-cli 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,13 +10,36 @@ export interface HttpRequest {
10
10
  timeoutMs?: number;
11
11
  /** Hard cap on the response body size in bytes; the request aborts if exceeded. */
12
12
  maxResponseBytes?: number;
13
+ /**
14
+ * Aborted when the engine's overall deadline (`timeoutMs`) passes. A transport should
15
+ * stop the request then (`fetch(url, { signal })`); the engine rejects at the deadline
16
+ * either way, and enforces `maxResponseBytes` on the body it gets back, so neither
17
+ * limit depends on the transport.
18
+ */
19
+ signal?: AbortSignal;
20
+ /**
21
+ * Always `"manual"` from the engine: a transport must not follow redirects. The
22
+ * engine follows them itself and decides per hop whether the `Authorization`
23
+ * header (the base URL's userinfo) goes along (same origin only). A fetch-based
24
+ * transport passes it on: `fetch(url, { redirect })`.
25
+ */
26
+ redirect?: "manual";
13
27
  }
14
28
  export interface HttpResponse {
15
29
  status: number;
16
30
  headers: http.IncomingHttpHeaders;
17
31
  body: Buffer;
32
+ /**
33
+ * The URL the response came from, if the transport knows it (fetch's
34
+ * `response.url`). When it is on another origin than the request, the transport
35
+ * followed a redirect itself and the engine rejects the response with a
36
+ * CkanNetworkError.
37
+ */
38
+ url?: string;
18
39
  }
19
40
  export type Transport = (request: HttpRequest) => Promise<HttpResponse>;
41
+ /** The message for a body over the size cap, naming the option on both sides. */
42
+ export declare function sizeLimitMessage(maxBytes: number): string;
20
43
  /**
21
44
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
22
45
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -8,6 +8,10 @@
8
8
  import http from "node:http";
9
9
  import https from "node:https";
10
10
  import { CkanNetworkError, redactUrl } from "./errors.js";
11
+ /** The message for a body over the size cap, naming the option on both sides. */
12
+ export function sizeLimitMessage(maxBytes) {
13
+ return `Response exceeded the size limit of ${maxBytes} bytes (maxResponseBytes; --max-response-bytes on the CLI)`;
14
+ }
11
15
  /**
12
16
  * The longest delay Node's timers support (2^31 - 1 ms, about 24.8 days). A longer one
13
17
  * prints a TimeoutOverflowWarning and fires after 1 ms, so timeouts are capped here.
@@ -68,7 +72,7 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
68
72
  if (maxBytes !== undefined && received > maxBytes) {
69
73
  aborted = true;
70
74
  res.destroy();
71
- settleReject(new CkanNetworkError(`Response exceeded maxResponseBytes (${maxBytes})`));
75
+ settleReject(new CkanNetworkError(sizeLimitMessage(maxBytes)));
72
76
  return;
73
77
  }
74
78
  chunks.push(chunk);
@@ -109,6 +113,15 @@ export const nodeHttpTransport = (request) => new Promise((resolve, reject) => {
109
113
  // Don't let a pending deadline timer keep the event loop alive on its own.
110
114
  deadline.unref?.();
111
115
  }
116
+ if (request.signal !== undefined) {
117
+ const abort = () => {
118
+ req.destroy(new CkanNetworkError(`Request exceeded the ${request.timeoutMs ?? 0}ms deadline`));
119
+ };
120
+ if (request.signal.aborted)
121
+ abort();
122
+ else
123
+ request.signal.addEventListener("abort", abort, { once: true });
124
+ }
112
125
  req.on("error", (err) => {
113
126
  // A timeout destroy already passes an CkanNetworkError; don't double-wrap.
114
127
  settleReject(err instanceof CkanNetworkError ? err : new CkanNetworkError(err.message, { cause: err }));
@@ -1,11 +1,11 @@
1
- export { CkanClient, siteRoot } from "./client.js";
1
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
2
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
3
3
  export type { EngineOptions, RawResponse } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
7
7
  export type { QueryParams, QueryValue } from "./query.js";
8
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
8
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
9
9
  export { assertValid } from "./validate.js";
10
10
  export type { Problem } from "./validate.js";
11
11
  export { PORTALS } from "./portals-list.js";
@@ -1,9 +1,9 @@
1
1
  // Public entry point for the API client library.
2
- export { CkanClient, siteRoot } from "./client.js";
2
+ export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
3
3
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export { buildQueryString } from "./query.js";
6
- export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, } from "./errors.js";
6
+ export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
7
7
  export { assertValid } from "./validate.js";
8
8
  export { PORTALS } from "./portals-list.js";
9
9
  export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
@@ -1,5 +1,5 @@
1
1
  import { CkanClient } from "./client.js";
2
- import type { EngineOptions } from "./engine.js";
2
+ import { type EngineOptions } from "./engine.js";
3
3
  import type { Portal } from "./types.js";
4
4
  /**
5
5
  * One identity per portal, whatever form its URL was written in: host (without
@@ -1,7 +1,8 @@
1
1
  // Known CKAN portals: lookup and a short live check. The list itself is in
2
2
  // portals-list.ts, maintained with scripts/update-portals.ts.
3
3
  import { CkanClient, siteRoot } from "./client.js";
4
- import { CkanApiError, CkanNetworkError, CkanParseError } from "./errors.js";
4
+ import { validateBaseUrl } from "./engine.js";
5
+ import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
5
6
  import { PORTALS } from "./portals-list.js";
6
7
  import { assertValid, intRangeProblem } from "./validate.js";
7
8
  /**
@@ -17,6 +18,10 @@ export function portalKey(url) {
17
18
  }
18
19
  /** A portal from `list` by its id (any case) or by any form of its URL. */
19
20
  export function findPortal(idOrUrl, list) {
21
+ if (typeof idOrUrl !== "string")
22
+ throw new CkanValidationError("Invalid portal: Expected a string.");
23
+ if (!Array.isArray(list))
24
+ throw new CkanValidationError("Invalid portal list: Expected an array.");
20
25
  const wanted = idOrUrl.trim().toLowerCase();
21
26
  const byId = list.find((p) => p.id.toLowerCase() === wanted);
22
27
  if (byId)
@@ -28,7 +33,14 @@ export function findPortal(idOrUrl, list) {
28
33
  catch {
29
34
  return undefined; // not a URL either
30
35
  }
31
- return list.find((p) => portalKey(p.url) === key);
36
+ return list.find((p) => {
37
+ try {
38
+ return portalKey(p.url) === key;
39
+ }
40
+ catch {
41
+ return false; // an entry without a usable url matches nothing
42
+ }
43
+ });
32
44
  }
33
45
  /**
34
46
  * A short live check of the CKAN portal a client points at. It works when `package_search?rows=0`
@@ -64,6 +76,17 @@ export async function checkPortal(client) {
64
76
  }
65
77
  return result;
66
78
  }
79
+ /**
80
+ * The first error `code` (`ENOTFOUND`, `ECONNREFUSED`, …) along a cause chain: the
81
+ * engine wraps a transport failure in a CkanNetworkError that names the request, so
82
+ * Node's code sits one or more levels down.
83
+ */
84
+ function errorCode(cause, depth = 0) {
85
+ if (typeof cause !== "object" || cause === null || depth > 5)
86
+ return undefined;
87
+ const code = cause.code;
88
+ return code !== undefined ? code : errorCode(cause.cause, depth + 1);
89
+ }
67
90
  /** A short, stable reason for a failed check. */
68
91
  function problemOf(err) {
69
92
  if (err instanceof CkanApiError) {
@@ -81,7 +104,7 @@ function problemOf(err) {
81
104
  return "invalid JSON";
82
105
  }
83
106
  if (err instanceof CkanNetworkError) {
84
- const code = err.cause?.code;
107
+ const code = errorCode(err.cause);
85
108
  if (code === "ENOTFOUND" || code === "EAI_AGAIN")
86
109
  return "host not found";
87
110
  if (code === "ECONNREFUSED")
@@ -92,6 +115,10 @@ function problemOf(err) {
92
115
  return "TLS certificate not verifiable";
93
116
  if (/timed out|deadline/.test(message))
94
117
  return "timeout";
118
+ // The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
119
+ const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
120
+ if (reason !== undefined)
121
+ return reason.slice(0, 80);
95
122
  }
96
123
  return message.slice(0, 80);
97
124
  }
@@ -133,6 +160,8 @@ const realSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
133
160
  * CkanValidationError for a bad `concurrency` or `retryDelayMs`, before any request.
134
161
  */
135
162
  export async function checkPortalUrls(urls, options = {}) {
163
+ if (!Array.isArray(urls))
164
+ throw new CkanValidationError("Invalid urls: Expected an array.");
136
165
  const concurrency = assertValid("concurrency", options.concurrency ?? DEFAULT_CHECK_CONCURRENCY, intRangeProblem(1, Number.MAX_SAFE_INTEGER));
137
166
  const { retryDelayMs } = options;
138
167
  if (retryDelayMs !== undefined)
@@ -142,6 +171,10 @@ export async function checkPortalUrls(urls, options = {}) {
142
171
  return mapLimit(urls, concurrency, async (url) => {
143
172
  let client;
144
173
  try {
174
+ // Checked here, not left to the client: `baseUrl: undefined` selects the
175
+ // Hamburg default, so an entry without a url was checked against Hamburg and
176
+ // reported as working, with Hamburg's dataset count.
177
+ validateBaseUrl(url);
145
178
  client = createClient({ ...options.engineOptions, baseUrl: url });
146
179
  }
147
180
  catch (err) {
@@ -160,6 +193,8 @@ export async function checkPortalUrls(urls, options = {}) {
160
193
  * What `ckan portals --check` prints.
161
194
  */
162
195
  export async function checkPortals(portals = PORTALS, options = {}) {
196
+ if (!Array.isArray(portals))
197
+ throw new CkanValidationError("Invalid portals: Expected an array.");
163
198
  const date = options.date ?? new Date().toISOString().slice(0, 10);
164
199
  const checks = await checkPortalUrls(portals.map((p) => p.url), options);
165
200
  return portals.map((portal, i) => withCheck(portal, checks[i], date));
@@ -15,6 +15,16 @@ export declare function isBlank(value: string): boolean;
15
15
  * silently return the unfiltered result. Values that are not strings pass.
16
16
  */
17
17
  export declare function blankProblem(value: unknown): string | undefined;
18
+ /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
19
+ export declare function textProblem(value: unknown): string | undefined;
20
+ /** A list of non-blank strings (filter queries, facet fields). */
21
+ export declare function textListProblem(value: unknown): string | undefined;
22
+ /**
23
+ * A value the query-string builder can send: a string, a finite number, a boolean
24
+ * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
25
+ * function or `NaN` would go out as `[object Object]` or `NaN`.
26
+ */
27
+ export declare function queryValueProblem(value: unknown): string | undefined;
18
28
  /**
19
29
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
20
30
  * A negative, fractional or non-finite value would go upstream as given, and
@@ -47,7 +57,9 @@ export declare function headerValueProblem(value: string): string | undefined;
47
57
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
48
58
  * transport);
49
59
  * - no query string or fragment: the API path is appended, so it would land in
50
- * front of `/api/3/action`.
60
+ * front of `/api/3/action`;
61
+ * - a `%` in the user name or password must start a valid escape (`%25` for a
62
+ * literal one): the engine decodes the userinfo for the Authorization header.
51
63
  *
52
64
  * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
53
65
  * reasons never quote the value, so a credential in it cannot leak through them.
@@ -30,6 +30,36 @@ export function blankProblem(value) {
30
30
  return "Expected a non-empty value.";
31
31
  return undefined;
32
32
  }
33
+ /** A non-blank string (an id, a query, a sort expression); anything else is refused. */
34
+ export function textProblem(value) {
35
+ if (typeof value !== "string")
36
+ return "Expected a string.";
37
+ return blankProblem(value);
38
+ }
39
+ /** A list of non-blank strings (filter queries, facet fields). */
40
+ export function textListProblem(value) {
41
+ if (!Array.isArray(value))
42
+ return "Expected a list of strings.";
43
+ if (value.some((v) => typeof v !== "string"))
44
+ return "Expected a list of strings.";
45
+ return blankProblem(value);
46
+ }
47
+ /**
48
+ * A value the query-string builder can send: a string, a finite number, a boolean
49
+ * or a Date, or a list of them (`null`/`undefined` mean "not given"). An object, a
50
+ * function or `NaN` would go out as `[object Object]` or `NaN`.
51
+ */
52
+ export function queryValueProblem(value) {
53
+ const ok = (v) => v === null ||
54
+ v === undefined ||
55
+ typeof v === "string" ||
56
+ typeof v === "boolean" ||
57
+ (typeof v === "number" && Number.isFinite(v)) ||
58
+ (v instanceof Date && !Number.isNaN(v.getTime()));
59
+ if (Array.isArray(value) ? value.every(ok) : ok(value))
60
+ return blankProblem(value);
61
+ return "Expected a string, a finite number, a boolean or a Date (or a list of them).";
62
+ }
33
63
  /**
34
64
  * A count or offset (`rows`, `start`, `offset`): a non-negative safe integer.
35
65
  * A negative, fractional or non-finite value would go upstream as given, and
@@ -65,6 +95,8 @@ export function intRangeProblem(min, max) {
65
95
  * so the source stays free of control bytes.
66
96
  */
67
97
  export function headerValueProblem(value) {
98
+ if (typeof value !== "string")
99
+ return "Expected a string.";
68
100
  const blank = blankProblem(value);
69
101
  if (blank !== undefined)
70
102
  return blank;
@@ -88,12 +120,16 @@ export function headerValueProblem(value) {
88
120
  * - an absolute URL with an http(s) scheme (`file:`, `ftp:` … never reach a
89
121
  * transport);
90
122
  * - no query string or fragment: the API path is appended, so it would land in
91
- * front of `/api/3/action`.
123
+ * front of `/api/3/action`;
124
+ * - a `%` in the user name or password must start a valid escape (`%25` for a
125
+ * literal one): the engine decodes the userinfo for the Authorization header.
92
126
  *
93
127
  * Userinfo (`https://user:pw@host`) is allowed; error messages redact it. The
94
128
  * reasons never quote the value, so a credential in it cannot leak through them.
95
129
  */
96
130
  export function baseUrlProblem(value) {
131
+ if (typeof value !== "string")
132
+ return "Expected a string with an absolute http(s) URL.";
97
133
  if (isBlank(value))
98
134
  return "Expected an absolute http(s) URL.";
99
135
  if (value !== value.trim())
@@ -115,5 +151,15 @@ export function baseUrlProblem(value) {
115
151
  }
116
152
  if (/[?#]/.test(value))
117
153
  return "Expected a site URL without a query string or fragment.";
154
+ // The engine decodes the userinfo into the Authorization header; a "%" that isn't an
155
+ // escape would fail there ("URI malformed") at request time. Reject it here.
156
+ for (const part of [url.username, url.password]) {
157
+ try {
158
+ decodeURIComponent(part);
159
+ }
160
+ catch {
161
+ return 'The user name or password has a "%" that is not followed by two hex digits; write a literal "%" as %25.';
162
+ }
163
+ }
118
164
  return undefined;
119
165
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.1.0",
3
+ "version": "0.2.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {
@@ -21,7 +21,7 @@
21
21
  "CONTRIBUTING.md"
22
22
  ],
23
23
  "engines": {
24
- "node": ">=20"
24
+ "node": ">=22.12"
25
25
  },
26
26
  "scripts": {
27
27
  "clean": "node -e \"require('fs').rmSync('dist',{recursive:true,force:true})\"",
@@ -66,7 +66,7 @@
66
66
  "commander": "15.0.0"
67
67
  },
68
68
  "devDependencies": {
69
- "@types/node": "^26.5.1",
69
+ "@types/node": "^22.20.5",
70
70
  "typescript": "7.0.2"
71
71
  }
72
72
  }