@maschinenlesbar.org/ckan-cli 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/DATA_LICENSE.md +78 -0
- package/README.md +36 -9
- package/dist/src/cli/commands/portal.js +12 -3
- package/dist/src/cli/index.js +8 -3
- package/dist/src/cli/io.d.ts +35 -6
- package/dist/src/cli/io.js +62 -10
- package/dist/src/cli/log.d.ts +95 -0
- package/dist/src/cli/log.js +159 -0
- package/dist/src/cli/program.d.ts +7 -0
- package/dist/src/cli/program.js +72 -2
- package/dist/src/cli/run.d.ts +35 -10
- package/dist/src/cli/run.js +175 -25
- package/dist/src/cli/shared.d.ts +12 -2
- package/dist/src/cli/shared.js +33 -4
- package/dist/src/client/client.js +4 -3
- package/dist/src/client/engine.d.ts +36 -1
- package/dist/src/client/engine.js +71 -9
- package/dist/src/client/errors.d.ts +51 -7
- package/dist/src/client/errors.js +100 -16
- package/dist/src/client/index.d.ts +3 -3
- package/dist/src/client/index.js +2 -2
- package/dist/src/client/portals.js +3 -3
- package/dist/src/client/validate.js +2 -2
- package/package.json +4 -3
package/dist/src/cli/shared.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// Shared helpers used across CLI command groups: option parsers, the global
|
|
2
2
|
// option resolver, and the JSON result renderer.
|
|
3
3
|
import { InvalidArgumentError } from "commander";
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
4
|
+
import { logOf } from "./io.js";
|
|
5
|
+
import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "../client/engine.js";
|
|
6
|
+
import { CkanError, cutForMessage, redactUrl } from "../client/errors.js";
|
|
6
7
|
import { findPortal } from "../client/portals.js";
|
|
7
8
|
import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
|
|
8
9
|
import { PORTALS } from "../client/portals-list.js";
|
|
@@ -104,7 +105,7 @@ export function parseBaseUrl(value) {
|
|
|
104
105
|
export function parsePortal(value) {
|
|
105
106
|
const portal = findPortal(value, PORTALS);
|
|
106
107
|
if (!portal) {
|
|
107
|
-
throw new InvalidArgumentError(`Unknown portal "${value}". \`ckan portals\` lists the known ones.`);
|
|
108
|
+
throw new InvalidArgumentError(`Unknown portal "${cutForMessage(redactUrl(value))}". \`ckan portals\` lists the known ones.`);
|
|
108
109
|
}
|
|
109
110
|
return portal.url;
|
|
110
111
|
}
|
|
@@ -169,6 +170,31 @@ export function renderJson(deps, global, value) {
|
|
|
169
170
|
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
|
170
171
|
deps.io.out(text);
|
|
171
172
|
}
|
|
173
|
+
/**
|
|
174
|
+
* Log a warning (`ckan.http`) when the effective base URL (--portal >
|
|
175
|
+
* --base-url > CKAN_BASE_URL > default) is plain `http:` to a host other than loopback
|
|
176
|
+
* (cleartextProblem): requests, and any credentials in the URL, travel unencrypted.
|
|
177
|
+
* Called once per run, after the options are parsed and before the first request;
|
|
178
|
+
* stdout and the exit code are untouched.
|
|
179
|
+
*/
|
|
180
|
+
export function warnOnCleartext(deps, global) {
|
|
181
|
+
const problem = cleartextProblem(global.portal ?? global.baseUrl ?? DEFAULT_BASE_URL);
|
|
182
|
+
if (problem !== undefined)
|
|
183
|
+
logOf(deps).warn("http", problem);
|
|
184
|
+
}
|
|
185
|
+
/** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
|
|
186
|
+
export function retryMessage(event) {
|
|
187
|
+
let host;
|
|
188
|
+
try {
|
|
189
|
+
host = new URL(event.url).host;
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
host = "the server";
|
|
193
|
+
}
|
|
194
|
+
const why = event.status === undefined ? "connection reset" : `HTTP ${event.status}`;
|
|
195
|
+
const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
|
|
196
|
+
return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
|
|
197
|
+
}
|
|
172
198
|
/**
|
|
173
199
|
* Wrap an async command action with consistent global-option resolution and
|
|
174
200
|
* client construction. The callback receives a context (client + resolved global
|
|
@@ -182,7 +208,10 @@ export function action(deps, fn) {
|
|
|
182
208
|
const command = args[args.length - 1];
|
|
183
209
|
const positionals = args.slice(0, Math.max(0, args.length - 2));
|
|
184
210
|
const global = command.optsWithGlobals();
|
|
185
|
-
|
|
211
|
+
warnOnCleartext(deps, global);
|
|
212
|
+
const options = toEngineOptions(global);
|
|
213
|
+
options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
|
|
214
|
+
const client = deps.createClient(options);
|
|
186
215
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
187
216
|
};
|
|
188
217
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// CkanClient — a typed client over the open (no-auth) read endpoints of any CKAN
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
3
|
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
|
-
import {
|
|
4
|
+
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
5
|
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
7
|
/**
|
|
@@ -159,12 +159,13 @@ export class CkanClient {
|
|
|
159
159
|
// Another JSON API at the same path (or a proxy's JSON error page) is not an
|
|
160
160
|
// envelope; say so rather than "failed: undefined".
|
|
161
161
|
if (typeof env !== "object" || env === null || Array.isArray(env) || typeof env.success !== "boolean") {
|
|
162
|
-
throw new CkanParseError(`The answer to "${name}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
162
|
+
throw new CkanParseError(`The answer to "${cutForMessage(name)}" is not a CKAN Action API response; is ${redactUrl(this.#site)} a CKAN site?`);
|
|
163
163
|
}
|
|
164
164
|
if (!env.success) {
|
|
165
165
|
// A success:false envelope can come with HTTP 200, so it never passes the
|
|
166
166
|
// engine's error-detail sanitising: strip terminal controls here too.
|
|
167
|
-
|
|
167
|
+
const type = env.error?.__type;
|
|
168
|
+
throw new CkanActionError(`CKAN action "${cutForMessage(name)}" failed: ${cutForMessage(sanitizeServerText(this.#engine.scrub(describeCkanError(env.error))))}`, { action: name, errorType: typeof type === "string" ? cutForMessage(sanitizeServerText(type)) : undefined });
|
|
168
169
|
}
|
|
169
170
|
// `{"success": true}` without a result would print nothing useful (and the CLI
|
|
170
171
|
// would crash rendering `undefined`); CKAN always sends one, `null` included.
|
|
@@ -52,6 +52,24 @@ export interface EngineOptions {
|
|
|
52
52
|
maxResponseBytes?: number;
|
|
53
53
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
54
54
|
sleep?: (ms: number) => Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* Called once per retry, right before the backoff sleep, for each retried 429/503
|
|
57
|
+
* and reset connection; never when there is no retry. A throw is swallowed.
|
|
58
|
+
*/
|
|
59
|
+
onRetry?: (event: RetryEvent) => void;
|
|
60
|
+
}
|
|
61
|
+
/** What `EngineOptions.onRetry` is told about one retry. */
|
|
62
|
+
export interface RetryEvent {
|
|
63
|
+
/** Which retry this is, counting from 1. */
|
|
64
|
+
retry: number;
|
|
65
|
+
/** The most retries this request may make (`maxRetries`). */
|
|
66
|
+
maxRetries: number;
|
|
67
|
+
/** How long the engine waits before sending the request again. */
|
|
68
|
+
delayMs: number;
|
|
69
|
+
/** The HTTP status that caused the retry; absent for a reset connection. */
|
|
70
|
+
status?: number;
|
|
71
|
+
/** The URL being retried, userinfo redacted. */
|
|
72
|
+
url: string;
|
|
55
73
|
}
|
|
56
74
|
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
57
75
|
export declare const MAX_RETRIES = 10;
|
|
@@ -116,6 +134,19 @@ export declare function sanitizeServerText(text: string): string;
|
|
|
116
134
|
* default transport still re-checks the scheme on every hop.
|
|
117
135
|
*/
|
|
118
136
|
export declare function validateBaseUrl(raw: string): string;
|
|
137
|
+
/**
|
|
138
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
139
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
140
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
141
|
+
* where nothing leaves the machine.
|
|
142
|
+
*
|
|
143
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
144
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
145
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
146
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
147
|
+
* (once per run, before the first request).
|
|
148
|
+
*/
|
|
149
|
+
export declare function cleartextProblem(baseUrl: string, secrets?: readonly string[]): string | undefined;
|
|
119
150
|
export declare class RequestEngine {
|
|
120
151
|
#private;
|
|
121
152
|
private readonly transport;
|
|
@@ -126,9 +157,13 @@ export declare class RequestEngine {
|
|
|
126
157
|
private readonly maxRedirects;
|
|
127
158
|
private readonly maxResponseBytes;
|
|
128
159
|
private readonly sleep;
|
|
160
|
+
private readonly onRetry;
|
|
129
161
|
constructor(options?: EngineOptions);
|
|
162
|
+
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
163
|
+
private backOff;
|
|
130
164
|
/**
|
|
131
|
-
* `text` without the base URL's credentials (raw and percent-decoded
|
|
165
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
166
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
132
167
|
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
133
168
|
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
134
169
|
* messages built from server text.
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
import { TextDecoder } from "node:util";
|
|
5
5
|
import { MAX_TIMEOUT_MS, nodeHttpTransport, sizeLimitMessage } from "./http.js";
|
|
6
6
|
import { buildQueryString } from "./query.js";
|
|
7
|
-
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, redactCredentials, redactUrl, } from "./errors.js";
|
|
7
|
+
import { CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, cutForMessage, describeCkanError, echoedCredentialForms, redactCredentials, redactSecrets, redactUrl, } from "./errors.js";
|
|
8
8
|
import { assertValid, baseUrlProblem, headerValueProblem, intRangeProblem } from "./validate.js";
|
|
9
9
|
export const DEFAULT_BASE_URL = "https://suche.transparenz.hamburg.de";
|
|
10
10
|
const DEFAULT_USER_AGENT = "ckan-cli";
|
|
@@ -116,6 +116,39 @@ export function sanitizeServerText(text) {
|
|
|
116
116
|
export function validateBaseUrl(raw) {
|
|
117
117
|
return assertValid("base URL", raw, baseUrlProblem).replace(/\/+$/, "");
|
|
118
118
|
}
|
|
119
|
+
/** True for a loopback host: `localhost`, 127.0.0.0/8 or `::1` (as URL#hostname spells it). */
|
|
120
|
+
function isLoopbackHost(hostname) {
|
|
121
|
+
return hostname === "localhost" || hostname === "[::1]" || /^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
|
|
122
|
+
}
|
|
123
|
+
/**
|
|
124
|
+
* Whether requests to `baseUrl` would travel unencrypted, as one sentence for a warning,
|
|
125
|
+
* or `undefined` when they would not: for `https:`, for a
|
|
126
|
+
* URL that does not parse, and for a loopback host (`localhost`, 127.0.0.0/8, `::1`),
|
|
127
|
+
* where nothing leaves the machine.
|
|
128
|
+
*
|
|
129
|
+
* The sentence names the host (`url.host`: host and port, never the userinfo) and what
|
|
130
|
+
* secret travels with the requests: the base URL's credentials when it carries userinfo,
|
|
131
|
+
* and every phrase in `secrets` (noun phrases such as "the API key"). It never contains
|
|
132
|
+
* a password or key. The CLI logs it as a `WARN` record of `ckan.http` on stderr
|
|
133
|
+
* (once per run, before the first request).
|
|
134
|
+
*/
|
|
135
|
+
export function cleartextProblem(baseUrl, secrets = []) {
|
|
136
|
+
let url;
|
|
137
|
+
try {
|
|
138
|
+
url = new URL(baseUrl);
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
return undefined;
|
|
142
|
+
}
|
|
143
|
+
if (url.protocol !== "http:" || isLoopbackHost(url.hostname))
|
|
144
|
+
return undefined;
|
|
145
|
+
const userinfo = url.username !== "" || url.password !== "";
|
|
146
|
+
const phrases = [...secrets, ...(userinfo ? ["the base URL's credentials"] : [])];
|
|
147
|
+
if (phrases.length === 0)
|
|
148
|
+
return `requests to ${url.host} are sent unencrypted (http:, not https:)`;
|
|
149
|
+
const verb = phrases.length === 1 && !userinfo ? "is" : "are";
|
|
150
|
+
return `${phrases.join(" and ")} ${verb} sent unencrypted to ${url.host} (http:, not https:)`;
|
|
151
|
+
}
|
|
119
152
|
// The headers the engine sets itself, under the exact keys it uses. They are the
|
|
120
153
|
// only ones that follow a cross-origin redirect.
|
|
121
154
|
const ENGINE_HEADERS = new Set(["Accept", "User-Agent"]);
|
|
@@ -192,6 +225,12 @@ export class RequestEngine {
|
|
|
192
225
|
#baseUrl;
|
|
193
226
|
/** The base URL's userinfo, raw and percent-decoded, for scrubbing server and transport text. */
|
|
194
227
|
#credentials;
|
|
228
|
+
/**
|
|
229
|
+
* The forms a server echoes that userinfo back in (the Basic value, the decoded
|
|
230
|
+
* `user:password`, the password alone; `echoedCredentialForms`), longest first, so a
|
|
231
|
+
* password never leaves half of the `user:password` around it.
|
|
232
|
+
*/
|
|
233
|
+
#echoed;
|
|
195
234
|
transport;
|
|
196
235
|
userAgent;
|
|
197
236
|
timeoutMs;
|
|
@@ -200,6 +239,7 @@ export class RequestEngine {
|
|
|
200
239
|
maxRedirects;
|
|
201
240
|
maxResponseBytes;
|
|
202
241
|
sleep;
|
|
242
|
+
onRetry;
|
|
203
243
|
constructor(options = {}) {
|
|
204
244
|
// The raw value, checked before the slash strip (the engine glues it into
|
|
205
245
|
// every URL), and here rather than only in the default transport: a library
|
|
@@ -215,6 +255,9 @@ export class RequestEngine {
|
|
|
215
255
|
return [raw];
|
|
216
256
|
}
|
|
217
257
|
});
|
|
258
|
+
this.#echoed = credentialsIn(this.#baseUrl)
|
|
259
|
+
.flatMap(echoedCredentialForms)
|
|
260
|
+
.sort((a, b) => b.length - a.length);
|
|
218
261
|
this.transport = functionOption("transport", options.transport, nodeHttpTransport);
|
|
219
262
|
// Only undefined selects the default; a blank or unsendable value is refused.
|
|
220
263
|
this.userAgent =
|
|
@@ -228,15 +271,34 @@ export class RequestEngine {
|
|
|
228
271
|
this.maxResponseBytes =
|
|
229
272
|
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
230
273
|
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
274
|
+
this.onRetry =
|
|
275
|
+
options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
|
|
276
|
+
}
|
|
277
|
+
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
278
|
+
async backOff(attempt, delayMs, url, status) {
|
|
279
|
+
try {
|
|
280
|
+
this.onRetry?.({
|
|
281
|
+
retry: attempt,
|
|
282
|
+
maxRetries: this.maxRetries,
|
|
283
|
+
delayMs,
|
|
284
|
+
...(status !== undefined ? { status } : {}),
|
|
285
|
+
url: redactUrl(url),
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
// a logging hook is no reason to fail the request
|
|
290
|
+
}
|
|
291
|
+
await this.sleep(delayMs);
|
|
231
292
|
}
|
|
232
293
|
/**
|
|
233
|
-
* `text` without the base URL's credentials (raw and percent-decoded
|
|
294
|
+
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
295
|
+
* server echoes them back in: `echoedCredentialForms`): server text
|
|
234
296
|
* (an error body or a CKAN error that echoes the request URL) and transport text
|
|
235
297
|
* (fetch's "Failed to fetch <url>") can carry them. The client uses it for its own
|
|
236
298
|
* messages built from server text.
|
|
237
299
|
*/
|
|
238
300
|
scrub(text) {
|
|
239
|
-
return this.#credentials.length === 0 ? text : redactCredentials(text, this.#credentials);
|
|
301
|
+
return this.#credentials.length === 0 ? text : redactSecrets(redactCredentials(text, this.#credentials), this.#echoed);
|
|
240
302
|
}
|
|
241
303
|
/**
|
|
242
304
|
* A failure as the `cause` of the error the engine raises: the original when its
|
|
@@ -361,7 +423,7 @@ export class RequestEngine {
|
|
|
361
423
|
const finalUrl = response.url;
|
|
362
424
|
if (typeof finalUrl === "string" && finalUrl !== "" && originOf(finalUrl) !== originOf(url)) {
|
|
363
425
|
throw new CkanNetworkError(`${method} ${cutForMessage(redactUrl(url))} failed: the transport followed a redirect to another origin ` +
|
|
364
|
-
`(${sanitizeServerText(redactUrl(this.scrub(finalUrl)))}); a transport must not follow redirects ` +
|
|
426
|
+
`(${cutForMessage(sanitizeServerText(redactUrl(this.scrub(finalUrl))))}); a transport must not follow redirects ` +
|
|
365
427
|
`(HttpRequest.redirect is "manual").`);
|
|
366
428
|
}
|
|
367
429
|
const status = response.status;
|
|
@@ -384,7 +446,7 @@ export class RequestEngine {
|
|
|
384
446
|
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
385
447
|
// zero-delay burst against a server that had just asked for less load.
|
|
386
448
|
const backoff = this.retryDelayMs * attempt;
|
|
387
|
-
await this.
|
|
449
|
+
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
|
|
388
450
|
continue;
|
|
389
451
|
}
|
|
390
452
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
@@ -399,7 +461,7 @@ export class RequestEngine {
|
|
|
399
461
|
// Only http(s) is followed: a `file:`, `ftp:`, `data:` or `javascript:` target
|
|
400
462
|
// never reaches a transport (a custom one may not check the scheme).
|
|
401
463
|
if (nextUrl.protocol !== "http:" && nextUrl.protocol !== "https:") {
|
|
402
|
-
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${sanitizeServerText(nextUrl.protocol)}" ` +
|
|
464
|
+
throw new CkanNetworkError(`Refusing to follow redirect to unsupported protocol "${cutForMessage(sanitizeServerText(nextUrl.protocol))}" ` +
|
|
403
465
|
`for ${method} ${cutForMessage(redactUrl(url))}`);
|
|
404
466
|
}
|
|
405
467
|
// Credential-strip guard: if the redirect crosses origin (scheme + host +
|
|
@@ -435,7 +497,7 @@ export class RequestEngine {
|
|
|
435
497
|
const detail = target !== undefined
|
|
436
498
|
? `stopped after ${this.maxRedirects} redirects (a redirect loop?)`
|
|
437
499
|
: location
|
|
438
|
-
? `redirect to ${sanitizeServerText(location)} not followed`
|
|
500
|
+
? `redirect to ${cutForMessage(sanitizeServerText(location))} not followed`
|
|
439
501
|
: "redirect not followed (no Location header)";
|
|
440
502
|
throw new CkanApiError({ status, url, method, body: this.scrub(body.toString("utf8")), detail });
|
|
441
503
|
}
|
|
@@ -459,7 +521,7 @@ export class RequestEngine {
|
|
|
459
521
|
// path) typically answers with an HTML page and HTTP 200.
|
|
460
522
|
// The Content-Type is server text (latin-1 decoded, so a 0x9B byte is the
|
|
461
523
|
// 8-bit CSI U+009B): sanitise it before it reaches stderr.
|
|
462
|
-
const mediaType = sanitizeServerText(res.contentType.split(";")[0]);
|
|
524
|
+
const mediaType = cutForMessage(sanitizeServerText(res.contentType.split(";")[0]));
|
|
463
525
|
const message = /json/i.test(mediaType)
|
|
464
526
|
? `Invalid JSON from ${cutForMessage(redactUrl(res.url))}`
|
|
465
527
|
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
@@ -550,7 +612,7 @@ function decodeBody(body, contentType, url) {
|
|
|
550
612
|
decoder = new TextDecoder(charset);
|
|
551
613
|
}
|
|
552
614
|
catch {
|
|
553
|
-
throw new CkanParseError(`Unsupported response charset "${sanitizeServerText(charset)}" from ${cutForMessage(redactUrl(url))}.`);
|
|
615
|
+
throw new CkanParseError(`Unsupported response charset "${cutForMessage(sanitizeServerText(charset))}" from ${cutForMessage(redactUrl(url))}.`);
|
|
554
616
|
}
|
|
555
617
|
return decoder.decode(body);
|
|
556
618
|
}
|
|
@@ -7,14 +7,26 @@
|
|
|
7
7
|
*/
|
|
8
8
|
export declare function redactUrl(url: string): string;
|
|
9
9
|
/**
|
|
10
|
-
* The userinfo a URL
|
|
11
|
-
* `https://alice:pa#ss@host` — or `[]` when it carries none.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
10
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
11
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
12
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
13
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
14
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
15
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
15
16
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
16
17
|
*/
|
|
17
18
|
export declare function credentialsIn(value: string): string[];
|
|
19
|
+
/**
|
|
20
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
21
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
22
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
23
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
24
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
25
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
26
|
+
*/
|
|
27
|
+
export declare function echoedCredentialForms(userinfo: string): string[];
|
|
28
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
29
|
+
export declare function redactSecrets(text: string, secrets: readonly string[]): string;
|
|
18
30
|
/**
|
|
19
31
|
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
20
32
|
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
@@ -24,11 +36,26 @@ export declare function redactCredentials(text: string, credentials: readonly st
|
|
|
24
36
|
/**
|
|
25
37
|
* Longest URL or server text (in characters) an error message shows. A long search
|
|
26
38
|
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
27
|
-
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
39
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
40
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
41
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
42
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
28
43
|
*/
|
|
29
44
|
export declare const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
30
|
-
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
45
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
31
46
|
export declare function cutForMessage(text: string): string;
|
|
47
|
+
/**
|
|
48
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
49
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
50
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
51
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
52
|
+
*/
|
|
53
|
+
export declare function cutText(text: string, max: number): string;
|
|
54
|
+
/**
|
|
55
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
56
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
57
|
+
*/
|
|
58
|
+
export declare function toWellFormed(text: string): string;
|
|
32
59
|
/** Base class for every error originating from this client. */
|
|
33
60
|
export declare class CkanError extends Error {
|
|
34
61
|
constructor(message: string, options?: {
|
|
@@ -66,6 +93,23 @@ export declare class CkanApiError extends CkanError {
|
|
|
66
93
|
/** True for statuses the API documents as transient and retry-able. */
|
|
67
94
|
get isRetryable(): boolean;
|
|
68
95
|
}
|
|
96
|
+
/**
|
|
97
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
98
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
99
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
100
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
101
|
+
* or undefined when the error carries none.
|
|
102
|
+
*/
|
|
103
|
+
export declare class CkanActionError extends CkanError {
|
|
104
|
+
readonly action: string;
|
|
105
|
+
readonly errorType: string | undefined;
|
|
106
|
+
constructor(message: string, args: {
|
|
107
|
+
action: string;
|
|
108
|
+
errorType?: string | undefined;
|
|
109
|
+
});
|
|
110
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
111
|
+
get isNotFound(): boolean;
|
|
112
|
+
}
|
|
69
113
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
70
114
|
export declare class CkanNetworkError extends CkanError {
|
|
71
115
|
}
|
|
@@ -16,7 +16,8 @@ export function redactUrl(url) {
|
|
|
16
16
|
// A value that doesn't parse can still carry credentials: cut them out by text.
|
|
17
17
|
return redactCredentials(url, credentialsIn(url));
|
|
18
18
|
}
|
|
19
|
-
// `user:pw@host` without a scheme parses as a URL with
|
|
19
|
+
// A URL without userinfo, or `user:pw@host` without a scheme (it parses as a URL with
|
|
20
|
+
// the scheme "user:"), which is no URL with credentials at all.
|
|
20
21
|
if (parsed.username === "" && parsed.password === "")
|
|
21
22
|
return redactCredentials(url, credentialsIn(url));
|
|
22
23
|
parsed.username = "***";
|
|
@@ -24,24 +25,22 @@ export function redactUrl(url) {
|
|
|
24
25
|
return parsed.href;
|
|
25
26
|
}
|
|
26
27
|
/**
|
|
27
|
-
* The userinfo a URL
|
|
28
|
-
* `https://alice:pa#ss@host` — or `[]` when it carries none.
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* The userinfo a URL carries, exactly as written — `["alice:pa#ss"]` for
|
|
29
|
+
* `https://alice:pa#ss@host` — or `[]` when it carries none. Only a value that starts
|
|
30
|
+
* with a scheme (`^[A-Za-z][A-Za-z0-9+.-]*://`) counts: a bare `a:b@c` is a dataset id, a
|
|
31
|
+
* search text or a User-Agent as often as a credential, and the base URL always has a
|
|
32
|
+
* scheme. It works on URLs that don't parse too: the userinfo is everything between
|
|
33
|
+
* `://` and the last `@` before the host. Used to redact those exact strings from text
|
|
32
34
|
* that echoes the value (usage errors, help), whatever characters the password contains.
|
|
33
35
|
*/
|
|
34
36
|
export function credentialsIn(value) {
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
// Without a scheme only the unmistakable `user:password@host` form counts.
|
|
38
|
-
if (schemeAt < 0 && !/^[^\s/@:]+:[^@]*@[^@\s/]/.test(rest))
|
|
37
|
+
const scheme = /^[A-Za-z][A-Za-z0-9+.-]*:\/\//.exec(value);
|
|
38
|
+
if (scheme === null)
|
|
39
39
|
return [];
|
|
40
|
-
|
|
41
|
-
const scheme = schemeAt >= 0 ? /[a-z][a-z0-9+.-]*$/i.exec(value.slice(0, schemeAt)) : null;
|
|
40
|
+
const rest = value.slice(scheme[0].length);
|
|
42
41
|
let parses = false;
|
|
43
42
|
try {
|
|
44
|
-
new URL(
|
|
43
|
+
new URL(value);
|
|
45
44
|
parses = true;
|
|
46
45
|
}
|
|
47
46
|
catch {
|
|
@@ -53,6 +52,46 @@ export function credentialsIn(value) {
|
|
|
53
52
|
const end = authority.lastIndexOf("@");
|
|
54
53
|
return end > 0 ? [rest.slice(0, end)] : [];
|
|
55
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* The forms in which a server may echo the credentials of a userinfo (`user:password`,
|
|
57
|
+
* as `credentialsIn` returns it) back in an error body: the `Authorization: Basic` value
|
|
58
|
+
* (base64 of the decoded `user:password`, UTF-8 as the engine sends it), the decoded
|
|
59
|
+
* `user:password` itself, and the password alone when it is at least 4 characters long.
|
|
60
|
+
* `[]` for a userinfo without a password. None of them has an `@` to anchor on, so they
|
|
61
|
+
* are replaced as exact strings (`redactSecrets`).
|
|
62
|
+
*/
|
|
63
|
+
export function echoedCredentialForms(userinfo) {
|
|
64
|
+
const colon = userinfo.indexOf(":");
|
|
65
|
+
if (colon < 0)
|
|
66
|
+
return [];
|
|
67
|
+
const decode = (part) => {
|
|
68
|
+
try {
|
|
69
|
+
return decodeURIComponent(part);
|
|
70
|
+
}
|
|
71
|
+
catch {
|
|
72
|
+
return part;
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
const user = decode(userinfo.slice(0, colon));
|
|
76
|
+
const password = decode(userinfo.slice(colon + 1));
|
|
77
|
+
if (password === "")
|
|
78
|
+
return [];
|
|
79
|
+
const pair = `${user}:${password}`;
|
|
80
|
+
const forms = [Buffer.from(pair, "utf8").toString("base64"), pair];
|
|
81
|
+
if (password.length >= 4)
|
|
82
|
+
forms.push(password);
|
|
83
|
+
return forms;
|
|
84
|
+
}
|
|
85
|
+
/** `text` with every occurrence of each of `secrets` (exact strings, no anchor) replaced by `***`. */
|
|
86
|
+
export function redactSecrets(text, secrets) {
|
|
87
|
+
let out = text;
|
|
88
|
+
for (const secret of secrets) {
|
|
89
|
+
if (secret === "")
|
|
90
|
+
continue;
|
|
91
|
+
out = out.split(secret).join("***");
|
|
92
|
+
}
|
|
93
|
+
return out;
|
|
94
|
+
}
|
|
56
95
|
/**
|
|
57
96
|
* `text` with every occurrence of each credential (as `credentialsIn` returns them) that is
|
|
58
97
|
* followed by `@` replaced by `***`. Matching the exact strings, not a pattern, covers
|
|
@@ -70,12 +109,37 @@ export function redactCredentials(text, credentials) {
|
|
|
70
109
|
/**
|
|
71
110
|
* Longest URL or server text (in characters) an error message shows. A long search
|
|
72
111
|
* query or a 200 kB error page would otherwise put one huge line on stderr or in a CI
|
|
73
|
-
* log. The error's `url`, `detail` and `body` properties keep the full value.
|
|
112
|
+
* log. The error's `url`, `detail` and `body` properties keep the full value. Every
|
|
113
|
+
* value an own message quotes from a server answer or the user's input (a redirect
|
|
114
|
+
* target, a charset, a Content-Type, an action name, a typed option value) is cut at
|
|
115
|
+
* this length too, so a library caller's `err.message` stays bounded.
|
|
74
116
|
*/
|
|
75
117
|
export const MAX_MESSAGE_VALUE_LENGTH = 500;
|
|
76
|
-
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters, ending in "…" when cut. */
|
|
118
|
+
/** `text` cut to MAX_MESSAGE_VALUE_LENGTH characters (never inside a surrogate pair), ending in "…" when cut. */
|
|
77
119
|
export function cutForMessage(text) {
|
|
78
|
-
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${text
|
|
120
|
+
return text.length > MAX_MESSAGE_VALUE_LENGTH ? `${cutText(text, MAX_MESSAGE_VALUE_LENGTH)}…` : text;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* `text` cut to at most `max` UTF-16 units, never inside a surrogate pair: when the cut
|
|
124
|
+
* would land after a high surrogate it is made one unit earlier, so a message that holds
|
|
125
|
+
* the cut text is well-formed (a lone `\ud83d` makes jq reject a whole JSON stream).
|
|
126
|
+
* Text no longer than `max` is returned as it is; the caller marks a cut.
|
|
127
|
+
*/
|
|
128
|
+
export function cutText(text, max) {
|
|
129
|
+
if (text.length <= max)
|
|
130
|
+
return text;
|
|
131
|
+
const end = max > 0 && isHighSurrogate(text.charCodeAt(max - 1)) ? max - 1 : max;
|
|
132
|
+
return text.slice(0, end);
|
|
133
|
+
}
|
|
134
|
+
function isHighSurrogate(c) {
|
|
135
|
+
return c >= 0xd800 && c <= 0xdbff;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* `text` with every lone surrogate (half of a character) replaced by U+FFFD, like
|
|
139
|
+
* `String.prototype.toWellFormed` (ES2024, so not in this package's `lib`).
|
|
140
|
+
*/
|
|
141
|
+
export function toWellFormed(text) {
|
|
142
|
+
return text.replace(/[\ud800-\udbff](?![\udc00-\udfff])|(?<![\ud800-\udbff])[\udc00-\udfff]/g, "\ufffd");
|
|
79
143
|
}
|
|
80
144
|
/** Base class for every error originating from this client. */
|
|
81
145
|
export class CkanError extends Error {
|
|
@@ -134,6 +198,26 @@ export class CkanApiError extends CkanError {
|
|
|
134
198
|
return this.status === 429 || this.status === 503;
|
|
135
199
|
}
|
|
136
200
|
}
|
|
201
|
+
/**
|
|
202
|
+
* CKAN answered with a `success: false` envelope on a 2xx status: the portal's own error
|
|
203
|
+
* answer, which CKAN can send with HTTP 200 (an error status is a `CkanApiError`). The
|
|
204
|
+
* message is `CKAN action "<action>" failed: <CKAN's error, one line>`; `errorType` is
|
|
205
|
+
* CKAN's `__type` (`"Not Found Error"`, `"Authorization Error"`, `"Validation Error"`),
|
|
206
|
+
* or undefined when the error carries none.
|
|
207
|
+
*/
|
|
208
|
+
export class CkanActionError extends CkanError {
|
|
209
|
+
action;
|
|
210
|
+
errorType;
|
|
211
|
+
constructor(message, args) {
|
|
212
|
+
super(message);
|
|
213
|
+
this.action = args.action;
|
|
214
|
+
this.errorType = args.errorType;
|
|
215
|
+
}
|
|
216
|
+
/** True for CKAN's `Not Found Error`: what a 404 says, sent with HTTP 200. */
|
|
217
|
+
get isNotFound() {
|
|
218
|
+
return this.errorType === "Not Found Error";
|
|
219
|
+
}
|
|
220
|
+
}
|
|
137
221
|
/** A transport-level failure (DNS, connection reset, timeout, ...). */
|
|
138
222
|
export class CkanNetworkError extends CkanError {
|
|
139
223
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
3
|
-
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
2
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
3
|
+
export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
8
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
9
|
export { assertValid } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
package/dist/src/client/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
// Public entry point for the API client library.
|
|
2
2
|
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
3
|
-
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, } from "./engine.js";
|
|
3
|
+
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
|
-
export { CkanError, CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
6
|
+
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
7
|
export { assertValid } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// portals-list.ts, maintained with scripts/update-portals.ts.
|
|
3
3
|
import { CkanClient, siteRoot } from "./client.js";
|
|
4
4
|
import { validateBaseUrl } from "./engine.js";
|
|
5
|
-
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError } from "./errors.js";
|
|
5
|
+
import { CkanApiError, CkanNetworkError, CkanParseError, CkanValidationError, cutText } from "./errors.js";
|
|
6
6
|
import { PORTALS } from "./portals-list.js";
|
|
7
7
|
import { assertValid, intRangeProblem } from "./validate.js";
|
|
8
8
|
/**
|
|
@@ -118,9 +118,9 @@ function problemOf(err) {
|
|
|
118
118
|
// The engine's "GET <url> failed: <reason>": the reason is what tells portals apart.
|
|
119
119
|
const reason = /^[A-Z]+ \S+ failed: (.+)$/.exec(message)?.[1];
|
|
120
120
|
if (reason !== undefined)
|
|
121
|
-
return reason
|
|
121
|
+
return cutText(reason, 80);
|
|
122
122
|
}
|
|
123
|
-
return message
|
|
123
|
+
return cutText(message, 80);
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
126
|
* A portal entry updated with a check made on `date` (`YYYY-MM-DD`). A failed
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// the reason a value is invalid, or `undefined` when it is valid. Client methods
|
|
3
3
|
// enforce them with `assertValid` before any request; the CLI's value parsers call
|
|
4
4
|
// the same functions, so a rule is written once and the CLI and the library agree.
|
|
5
|
-
import { CkanValidationError } from "./errors.js";
|
|
5
|
+
import { CkanValidationError, cutForMessage } from "./errors.js";
|
|
6
6
|
/**
|
|
7
7
|
* Throw `CkanValidationError("Invalid <name>: <reason>")` when `problem` finds
|
|
8
8
|
* something wrong with `value`; otherwise return `value` unchanged. A method that
|
|
@@ -147,7 +147,7 @@ export function baseUrlProblem(value) {
|
|
|
147
147
|
return "Expected an absolute http(s) URL.";
|
|
148
148
|
}
|
|
149
149
|
if (url.protocol !== "http:" && url.protocol !== "https:") {
|
|
150
|
-
return `Unsupported scheme "${url.protocol}". Expected an http(s) URL.`;
|
|
150
|
+
return `Unsupported scheme "${cutForMessage(url.protocol)}". Expected an http(s) URL.`;
|
|
151
151
|
}
|
|
152
152
|
if (/[?#]/.test(value))
|
|
153
153
|
return "Expected a site URL without a query string or fragment.";
|