@maschinenlesbar.org/ckan-cli 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -294,7 +294,7 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
294
294
  | `--portal <id>` | a known portal by id (`ckan portals`); cannot be combined with `--base-url` |
295
295
  | `--timeout <ms>` | Per-request timeout (default `30000`; at most `2147483647`) |
296
296
  | `--user-agent <ua>` | `User-Agent` header value |
297
- | `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried |
297
+ | `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried. Each retry logs one WARN record of `ckan.http` before it waits (`HTTP 503 from host: retry 1 of 3 in 2 s`). |
298
298
  | `--max-response-bytes <n>` | Cap response body size in bytes (`0` = unlimited; default 100 MiB) |
299
299
 
300
300
  A base URL on plain `http:` (from `--base-url` or `CKAN_BASE_URL`) to a host other than
@@ -1,5 +1,5 @@
1
1
  import { type CliDeps } from "./io.js";
2
- import { type EngineOptions } from "../client/engine.js";
2
+ import { type EngineOptions, type RetryEvent } from "../client/engine.js";
3
3
  /** commander value-parser: a non-negative integer. */
4
4
  export declare function parseIntArg(value: string): number;
5
5
  /**
@@ -74,6 +74,8 @@ export declare function renderJson(deps: CliDeps, global: GlobalOptions, value:
74
74
  * stdout and the exit code are untouched.
75
75
  */
76
76
  export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
77
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
78
+ export declare function retryMessage(event: RetryEvent): string;
77
79
  export interface ActionContext {
78
80
  client: ReturnType<CliDeps["createClient"]>;
79
81
  global: GlobalOptions;
@@ -182,6 +182,19 @@ export function warnOnCleartext(deps, global) {
182
182
  if (problem !== undefined)
183
183
  logOf(deps).warn("http", problem);
184
184
  }
185
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
186
+ export function retryMessage(event) {
187
+ let host;
188
+ try {
189
+ host = new URL(event.url).host;
190
+ }
191
+ catch {
192
+ host = "the server";
193
+ }
194
+ const why = event.status === undefined ? "connection reset" : `HTTP ${event.status}`;
195
+ const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
196
+ return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
197
+ }
185
198
  /**
186
199
  * Wrap an async command action with consistent global-option resolution and
187
200
  * client construction. The callback receives a context (client + resolved global
@@ -196,7 +209,9 @@ export function action(deps, fn) {
196
209
  const positionals = args.slice(0, Math.max(0, args.length - 2));
197
210
  const global = command.optsWithGlobals();
198
211
  warnOnCleartext(deps, global);
199
- const client = deps.createClient(toEngineOptions(global));
212
+ const options = toEngineOptions(global);
213
+ options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
214
+ const client = deps.createClient(options);
200
215
  await fn({ client, global, opts: command.opts() }, positionals);
201
216
  };
202
217
  }
@@ -52,6 +52,24 @@ export interface EngineOptions {
52
52
  maxResponseBytes?: number;
53
53
  /** Injectable sleep, primarily for deterministic tests. */
54
54
  sleep?: (ms: number) => Promise<void>;
55
+ /**
56
+ * Called once per retry, right before the backoff sleep, for each retried 429/503
57
+ * and reset connection; never when there is no retry. A throw is swallowed.
58
+ */
59
+ onRetry?: (event: RetryEvent) => void;
60
+ }
61
+ /** What `EngineOptions.onRetry` is told about one retry. */
62
+ export interface RetryEvent {
63
+ /** Which retry this is, counting from 1. */
64
+ retry: number;
65
+ /** The most retries this request may make (`maxRetries`). */
66
+ maxRetries: number;
67
+ /** How long the engine waits before sending the request again. */
68
+ delayMs: number;
69
+ /** The HTTP status that caused the retry; absent for a reset connection. */
70
+ status?: number;
71
+ /** The URL being retried, userinfo redacted. */
72
+ url: string;
55
73
  }
56
74
  /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
57
75
  export declare const MAX_RETRIES = 10;
@@ -139,7 +157,10 @@ export declare class RequestEngine {
139
157
  private readonly maxRedirects;
140
158
  private readonly maxResponseBytes;
141
159
  private readonly sleep;
160
+ private readonly onRetry;
142
161
  constructor(options?: EngineOptions);
162
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
163
+ private backOff;
143
164
  /**
144
165
  * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
145
166
  * server echoes them back in: `echoedCredentialForms`): server text
@@ -239,6 +239,7 @@ export class RequestEngine {
239
239
  maxRedirects;
240
240
  maxResponseBytes;
241
241
  sleep;
242
+ onRetry;
242
243
  constructor(options = {}) {
243
244
  // The raw value, checked before the slash strip (the engine glues it into
244
245
  // every URL), and here rather than only in the default transport: a library
@@ -270,6 +271,24 @@ export class RequestEngine {
270
271
  this.maxResponseBytes =
271
272
  intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
272
273
  this.sleep = functionOption("sleep", options.sleep, realSleep);
274
+ this.onRetry =
275
+ options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
276
+ }
277
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
278
+ async backOff(attempt, delayMs, url, status) {
279
+ try {
280
+ this.onRetry?.({
281
+ retry: attempt,
282
+ maxRetries: this.maxRetries,
283
+ delayMs,
284
+ ...(status !== undefined ? { status } : {}),
285
+ url: redactUrl(url),
286
+ });
287
+ }
288
+ catch {
289
+ // a logging hook is no reason to fail the request
290
+ }
291
+ await this.sleep(delayMs);
273
292
  }
274
293
  /**
275
294
  * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
@@ -427,7 +446,7 @@ export class RequestEngine {
427
446
  // for less: `Retry-After: 0` or a date in the past turned the retries into a
428
447
  // zero-delay burst against a server that had just asked for less load.
429
448
  const backoff = this.retryDelayMs * attempt;
430
- await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
449
+ await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
431
450
  continue;
432
451
  }
433
452
  // Follow redirects, resolving the Location relative to the current URL.
@@ -1,6 +1,6 @@
1
1
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
2
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
3
- export type { EngineOptions, RawResponse } from "./engine.js";
3
+ export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@maschinenlesbar.org/ckan-cli",
3
- "version": "0.4.0",
3
+ "version": "0.5.0",
4
4
  "description": "TypeScript API client and CLI for any CKAN open-data portal (Action API v3), e.g. the Hamburg Transparenzportal",
5
5
  "type": "module",
6
6
  "bin": {