@maschinenlesbar.org/ckan-cli 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/cli/shared.d.ts +3 -1
- package/dist/src/cli/shared.js +16 -1
- package/dist/src/client/engine.d.ts +21 -0
- package/dist/src/client/engine.js +20 -1
- package/dist/src/client/index.d.ts +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -294,7 +294,7 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
|
|
|
294
294
|
| `--portal <id>` | a known portal by id (`ckan portals`); cannot be combined with `--base-url` |
|
|
295
295
|
| `--timeout <ms>` | Per-request timeout (default `30000`; at most `2147483647`) |
|
|
296
296
|
| `--user-agent <ua>` | `User-Agent` header value |
|
|
297
|
-
| `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried |
|
|
297
|
+
| `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried. Each retry logs one WARN record of `ckan.http` before it waits (`HTTP 503 from host: retry 1 of 3 in 2 s`). |
|
|
298
298
|
| `--max-response-bytes <n>` | Cap response body size in bytes (`0` = unlimited; default 100 MiB) |
|
|
299
299
|
|
|
300
300
|
A base URL on plain `http:` (from `--base-url` or `CKAN_BASE_URL`) to a host other than
|
package/dist/src/cli/shared.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type CliDeps } from "./io.js";
|
|
2
|
-
import { type EngineOptions } from "../client/engine.js";
|
|
2
|
+
import { type EngineOptions, type RetryEvent } from "../client/engine.js";
|
|
3
3
|
/** commander value-parser: a non-negative integer. */
|
|
4
4
|
export declare function parseIntArg(value: string): number;
|
|
5
5
|
/**
|
|
@@ -74,6 +74,8 @@ export declare function renderJson(deps: CliDeps, global: GlobalOptions, value:
|
|
|
74
74
|
* stdout and the exit code are untouched.
|
|
75
75
|
*/
|
|
76
76
|
export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
|
|
77
|
+
/** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
|
|
78
|
+
export declare function retryMessage(event: RetryEvent): string;
|
|
77
79
|
export interface ActionContext {
|
|
78
80
|
client: ReturnType<CliDeps["createClient"]>;
|
|
79
81
|
global: GlobalOptions;
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -182,6 +182,19 @@ export function warnOnCleartext(deps, global) {
|
|
|
182
182
|
if (problem !== undefined)
|
|
183
183
|
logOf(deps).warn("http", problem);
|
|
184
184
|
}
|
|
185
|
+
/** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
|
|
186
|
+
export function retryMessage(event) {
|
|
187
|
+
let host;
|
|
188
|
+
try {
|
|
189
|
+
host = new URL(event.url).host;
|
|
190
|
+
}
|
|
191
|
+
catch {
|
|
192
|
+
host = "the server";
|
|
193
|
+
}
|
|
194
|
+
const why = event.status === undefined ? "connection reset" : `HTTP ${event.status}`;
|
|
195
|
+
const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
|
|
196
|
+
return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
|
|
197
|
+
}
|
|
185
198
|
/**
|
|
186
199
|
* Wrap an async command action with consistent global-option resolution and
|
|
187
200
|
* client construction. The callback receives a context (client + resolved global
|
|
@@ -196,7 +209,9 @@ export function action(deps, fn) {
|
|
|
196
209
|
const positionals = args.slice(0, Math.max(0, args.length - 2));
|
|
197
210
|
const global = command.optsWithGlobals();
|
|
198
211
|
warnOnCleartext(deps, global);
|
|
199
|
-
const
|
|
212
|
+
const options = toEngineOptions(global);
|
|
213
|
+
options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
|
|
214
|
+
const client = deps.createClient(options);
|
|
200
215
|
await fn({ client, global, opts: command.opts() }, positionals);
|
|
201
216
|
};
|
|
202
217
|
}
|
|
@@ -52,6 +52,24 @@ export interface EngineOptions {
|
|
|
52
52
|
maxResponseBytes?: number;
|
|
53
53
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
54
54
|
sleep?: (ms: number) => Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* Called once per retry, right before the backoff sleep, for each retried 429/503
|
|
57
|
+
* and reset connection; never when there is no retry. A throw is swallowed.
|
|
58
|
+
*/
|
|
59
|
+
onRetry?: (event: RetryEvent) => void;
|
|
60
|
+
}
|
|
61
|
+
/** What `EngineOptions.onRetry` is told about one retry. */
|
|
62
|
+
export interface RetryEvent {
|
|
63
|
+
/** Which retry this is, counting from 1. */
|
|
64
|
+
retry: number;
|
|
65
|
+
/** The most retries this request may make (`maxRetries`). */
|
|
66
|
+
maxRetries: number;
|
|
67
|
+
/** How long the engine waits before sending the request again. */
|
|
68
|
+
delayMs: number;
|
|
69
|
+
/** The HTTP status that caused the retry; absent for a reset connection. */
|
|
70
|
+
status?: number;
|
|
71
|
+
/** The URL being retried, userinfo redacted. */
|
|
72
|
+
url: string;
|
|
55
73
|
}
|
|
56
74
|
/** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
|
|
57
75
|
export declare const MAX_RETRIES = 10;
|
|
@@ -139,7 +157,10 @@ export declare class RequestEngine {
|
|
|
139
157
|
private readonly maxRedirects;
|
|
140
158
|
private readonly maxResponseBytes;
|
|
141
159
|
private readonly sleep;
|
|
160
|
+
private readonly onRetry;
|
|
142
161
|
constructor(options?: EngineOptions);
|
|
162
|
+
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
163
|
+
private backOff;
|
|
143
164
|
/**
|
|
144
165
|
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
145
166
|
* server echoes them back in: `echoedCredentialForms`): server text
|
|
@@ -239,6 +239,7 @@ export class RequestEngine {
|
|
|
239
239
|
maxRedirects;
|
|
240
240
|
maxResponseBytes;
|
|
241
241
|
sleep;
|
|
242
|
+
onRetry;
|
|
242
243
|
constructor(options = {}) {
|
|
243
244
|
// The raw value, checked before the slash strip (the engine glues it into
|
|
244
245
|
// every URL), and here rather than only in the default transport: a library
|
|
@@ -270,6 +271,24 @@ export class RequestEngine {
|
|
|
270
271
|
this.maxResponseBytes =
|
|
271
272
|
intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
|
|
272
273
|
this.sleep = functionOption("sleep", options.sleep, realSleep);
|
|
274
|
+
this.onRetry =
|
|
275
|
+
options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
|
|
276
|
+
}
|
|
277
|
+
/** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
|
|
278
|
+
async backOff(attempt, delayMs, url, status) {
|
|
279
|
+
try {
|
|
280
|
+
this.onRetry?.({
|
|
281
|
+
retry: attempt,
|
|
282
|
+
maxRetries: this.maxRetries,
|
|
283
|
+
delayMs,
|
|
284
|
+
...(status !== undefined ? { status } : {}),
|
|
285
|
+
url: redactUrl(url),
|
|
286
|
+
});
|
|
287
|
+
}
|
|
288
|
+
catch {
|
|
289
|
+
// a logging hook is no reason to fail the request
|
|
290
|
+
}
|
|
291
|
+
await this.sleep(delayMs);
|
|
273
292
|
}
|
|
274
293
|
/**
|
|
275
294
|
* `text` without the base URL's credentials (raw and percent-decoded, and the forms a
|
|
@@ -427,7 +446,7 @@ export class RequestEngine {
|
|
|
427
446
|
// for less: `Retry-After: 0` or a date in the past turned the retries into a
|
|
428
447
|
// zero-delay burst against a server that had just asked for less load.
|
|
429
448
|
const backoff = this.retryDelayMs * attempt;
|
|
430
|
-
await this.
|
|
449
|
+
await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
|
|
431
450
|
continue;
|
|
432
451
|
}
|
|
433
452
|
// Follow redirects, resolving the Location relative to the current URL.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
|
|
2
2
|
export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
|
|
3
|
-
export type { EngineOptions, RawResponse } from "./engine.js";
|
|
3
|
+
export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
package/package.json
CHANGED