@maschinenlesbar.org/ckan-cli 0.4.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/README.md +2 -2
  2. package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/engine.d.ts +21 -0
  3. package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/engine.js +20 -1
  4. package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/index.d.ts +1 -1
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/package.json +32 -0
  6. package/dist/src/cli/commands/catalogue.js +1 -1
  7. package/dist/src/cli/commands/portal.js +4 -4
  8. package/dist/src/cli/io.d.ts +2 -2
  9. package/dist/src/cli/log.js +1 -1
  10. package/dist/src/cli/program.js +4 -4
  11. package/dist/src/cli/run.js +1 -1
  12. package/dist/src/cli/shared.d.ts +3 -1
  13. package/dist/src/cli/shared.js +21 -6
  14. package/dist/src/index.d.ts +1 -1
  15. package/dist/src/index.js +2 -1
  16. package/node_modules/commander/LICENSE +22 -0
  17. package/node_modules/commander/Readme.md +1172 -0
  18. package/node_modules/commander/index.js +21 -0
  19. package/node_modules/commander/lib/argument.js +147 -0
  20. package/node_modules/commander/lib/command.js +2790 -0
  21. package/node_modules/commander/lib/error.js +36 -0
  22. package/node_modules/commander/lib/help.js +731 -0
  23. package/node_modules/commander/lib/option.js +377 -0
  24. package/node_modules/commander/lib/suggestSimilar.js +99 -0
  25. package/node_modules/commander/package-support.json +19 -0
  26. package/node_modules/commander/package.json +64 -0
  27. package/node_modules/commander/typings/index.d.ts +1113 -0
  28. package/package.json +11 -16
  29. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/client.d.ts +0 -0
  30. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/client.js +0 -0
  31. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/errors.d.ts +0 -0
  32. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/errors.js +0 -0
  33. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/http.d.ts +0 -0
  34. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/http.js +0 -0
  35. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/index.js +0 -0
  36. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/portals-list.d.ts +0 -0
  37. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/portals-list.js +0 -0
  38. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/portals.d.ts +0 -0
  39. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/portals.js +0 -0
  40. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/query.d.ts +0 -0
  41. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/query.js +0 -0
  42. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/types.d.ts +0 -0
  43. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/types.js +0 -0
  44. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/validate.d.ts +0 -0
  45. /package/dist/{src/client → node_modules/@maschinenlesbar.org/ckan-client/dist/src}/validate.js +0 -0
package/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
  [![Release](https://github.com/maschinenlesbar-org/ckan-cli/actions/workflows/release.yml/badge.svg)](https://github.com/maschinenlesbar-org/ckan-cli/actions/workflows/release.yml)
5
5
  [![npm](https://img.shields.io/npm/v/@maschinenlesbar.org/ckan-cli)](https://www.npmjs.com/package/@maschinenlesbar.org/ckan-cli)
6
6
 
7
- **Website:** [English](https://maschinenlesbar-org.github.io/ckan-cli/) · [Deutsch](https://maschinenlesbar-org.github.io/ckan-cli/de/) — command reference, guides and API docs
7
+ **Website:** [English](https://maschinenlesbar.org/ckan-cli/) · [Deutsch](https://maschinenlesbar.org/ckan-cli/de/) — command reference, guides and API docs
8
8
 
9
9
  Search any [CKAN](https://ckan.org/) open-data portal from your terminal. `ckan`
10
10
  is a command-line tool over the CKAN Action API (`<portal>/api/3/action`), which
@@ -294,7 +294,7 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
294
294
  | `--portal <id>` | a known portal by id (`ckan portals`); cannot be combined with `--base-url` |
295
295
  | `--timeout <ms>` | Per-request timeout (default `30000`; at most `2147483647`) |
296
296
  | `--user-agent <ua>` | `User-Agent` header value |
297
- | `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried |
297
+ | `--max-retries <n>` | Retries for transient `429`/`503` responses (`0`–`10`, default `2`). Each retry backs off linearly (0.2 s, 0.4 s, …) or waits the server's `Retry-After` when that is longer, up to 30 s; a longer one is not retried, and the error names the wait. A reset or other network error is not retried. Each retry logs one WARN record of `ckan.http` before it waits (`HTTP 503 from host: retry 1 of 3 in 2 s`). |
298
298
  | `--max-response-bytes <n>` | Cap response body size in bytes (`0` = unlimited; default 100 MiB) |
299
299
 
300
300
  A base URL on plain `http:` (from `--base-url` or `CKAN_BASE_URL`) to a host other than
@@ -52,6 +52,24 @@ export interface EngineOptions {
52
52
  maxResponseBytes?: number;
53
53
  /** Injectable sleep, primarily for deterministic tests. */
54
54
  sleep?: (ms: number) => Promise<void>;
55
+ /**
56
+ * Called once per retry, right before the backoff sleep, for each retried 429/503
57
+ * and reset connection; never when there is no retry. A throw is swallowed.
58
+ */
59
+ onRetry?: (event: RetryEvent) => void;
60
+ }
61
+ /** What `EngineOptions.onRetry` is told about one retry. */
62
+ export interface RetryEvent {
63
+ /** Which retry this is, counting from 1. */
64
+ retry: number;
65
+ /** The most retries this request may make (`maxRetries`). */
66
+ maxRetries: number;
67
+ /** How long the engine waits before sending the request again. */
68
+ delayMs: number;
69
+ /** The HTTP status that caused the retry; absent for a reset connection. */
70
+ status?: number;
71
+ /** The URL being retried, userinfo redacted. */
72
+ url: string;
55
73
  }
56
74
  /** Most automatic retries the engine performs (`maxRetries`, the CLI's --max-retries). */
57
75
  export declare const MAX_RETRIES = 10;
@@ -139,7 +157,10 @@ export declare class RequestEngine {
139
157
  private readonly maxRedirects;
140
158
  private readonly maxResponseBytes;
141
159
  private readonly sleep;
160
+ private readonly onRetry;
142
161
  constructor(options?: EngineOptions);
162
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
163
+ private backOff;
143
164
  /**
144
165
  * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
145
166
  * server echoes them back in: `echoedCredentialForms`): server text
@@ -239,6 +239,7 @@ export class RequestEngine {
239
239
  maxRedirects;
240
240
  maxResponseBytes;
241
241
  sleep;
242
+ onRetry;
242
243
  constructor(options = {}) {
243
244
  // The raw value, checked before the slash strip (the engine glues it into
244
245
  // every URL), and here rather than only in the default transport: a library
@@ -270,6 +271,24 @@ export class RequestEngine {
270
271
  this.maxResponseBytes =
271
272
  intOption("maxResponseBytes", options.maxResponseBytes, 0, Number.MAX_SAFE_INTEGER) ?? DEFAULT_MAX_RESPONSE_BYTES;
272
273
  this.sleep = functionOption("sleep", options.sleep, realSleep);
274
+ this.onRetry =
275
+ options.onRetry === undefined ? undefined : functionOption("onRetry", options.onRetry, () => { });
276
+ }
277
+ /** Tell `onRetry` about a retry, then wait. A throwing callback never breaks the request. */
278
+ async backOff(attempt, delayMs, url, status) {
279
+ try {
280
+ this.onRetry?.({
281
+ retry: attempt,
282
+ maxRetries: this.maxRetries,
283
+ delayMs,
284
+ ...(status !== undefined ? { status } : {}),
285
+ url: redactUrl(url),
286
+ });
287
+ }
288
+ catch {
289
+ // a logging hook is no reason to fail the request
290
+ }
291
+ await this.sleep(delayMs);
273
292
  }
274
293
  /**
275
294
  * `text` without the base URL's credentials (raw and percent-decoded, and the forms a
@@ -427,7 +446,7 @@ export class RequestEngine {
427
446
  // for less: `Retry-After: 0` or a date in the past turned the retries into a
428
447
  // zero-delay burst against a server that had just asked for less load.
429
448
  const backoff = this.retryDelayMs * attempt;
430
- await this.sleep(retryAfter === undefined ? backoff : Math.max(retryAfter, backoff));
449
+ await this.backOff(attempt, retryAfter === undefined ? backoff : Math.max(retryAfter, backoff), url, status);
431
450
  continue;
432
451
  }
433
452
  // Follow redirects, resolving the Location relative to the current URL.
@@ -1,6 +1,6 @@
1
1
  export { ALL_FIELDS_PAGE_DELAY_MS, CkanClient, MAX_ALL_FIELDS_PAGES, siteRoot } from "./client.js";
2
2
  export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_RETRIES, MAX_RETRY_AFTER_MS, parseRetryAfter, validateBaseUrl, cleartextProblem, } from "./engine.js";
3
- export type { EngineOptions, RawResponse } from "./engine.js";
3
+ export type { EngineOptions, RawResponse, RetryEvent } from "./engine.js";
4
4
  export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
5
5
  export type { Transport, HttpRequest, HttpResponse } from "./http.js";
6
6
  export { buildQueryString } from "./query.js";
@@ -0,0 +1,32 @@
1
+ {
2
+ "name": "@maschinenlesbar.org/ckan-client",
3
+ "version": "0.0.1",
4
+ "private": true,
5
+ "description": "The typed API client of ckan-cli. Not published on its own: bundled into @maschinenlesbar.org/ckan-cli, whose library entry re-exports it.",
6
+ "type": "module",
7
+ "exports": {
8
+ ".": {
9
+ "types": "./dist/src/index.d.ts",
10
+ "import": "./dist/src/index.js"
11
+ },
12
+ "./internal/*": {
13
+ "types": "./dist/src/*.d.ts",
14
+ "import": "./dist/src/*.js"
15
+ },
16
+ "./package.json": "./package.json"
17
+ },
18
+ "files": [
19
+ "dist/src"
20
+ ],
21
+ "engines": {
22
+ "node": ">=22.12"
23
+ },
24
+ "license": "AGPL-3.0-or-later",
25
+ "author": "Sebastian Schürmann <sebs@2xs.org>",
26
+ "repository": {
27
+ "type": "git",
28
+ "url": "git+https://github.com/maschinenlesbar-org/ckan-cli.git",
29
+ "directory": "packages/client"
30
+ },
31
+ "homepage": "https://maschinenlesbar.org/ckan-cli/"
32
+ }
@@ -1,7 +1,7 @@
1
1
  // Commands over the catalogue: datasets, their resources, publishers, themes
2
2
  // and tags.
3
3
  import { InvalidArgumentError } from "commander";
4
- import { facetLimitProblem } from "../../client/validate.js";
4
+ import { facetLimitProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
5
5
  import { action, collectNonEmpty, once, parseBoundedInt, parseIntArg, parseNonEmpty, parseOptionValue, renderJson, } from "../shared.js";
6
6
  /**
7
7
  * commander value-parser for a list --limit: 1 or more. CKAN reads `limit=0` as
@@ -1,10 +1,10 @@
1
1
  // Commands about the portal itself rather than its datasets.
2
2
  import { InvalidArgumentError } from "commander";
3
3
  import { action, renderJson, toEngineOptions } from "../shared.js";
4
- import { checkPortals } from "../../client/portals.js";
5
- import { PORTALS } from "../../client/portals-list.js";
6
- import { isBlank } from "../../client/validate.js";
7
- import { cutForMessage, redactUrl } from "../../client/errors.js";
4
+ import { checkPortals } from "@maschinenlesbar.org/ckan-client/internal/portals";
5
+ import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
6
+ import { isBlank } from "@maschinenlesbar.org/ckan-client/internal/validate";
7
+ import { cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
8
8
  /**
9
9
  * A typed value as a message quotes it: cut at MAX_MESSAGE_VALUE_LENGTH, after its
10
10
  * credentials are redacted (a cut could otherwise leave part of a password without the
@@ -1,5 +1,5 @@
1
- import type { CkanClient } from "../client/client.js";
2
- import type { EngineOptions } from "../client/engine.js";
1
+ import type { CkanClient } from "@maschinenlesbar.org/ckan-client/internal/client";
2
+ import type { EngineOptions } from "@maschinenlesbar.org/ckan-client/internal/engine";
3
3
  import { type Logger } from "./log.js";
4
4
  export interface CliIO {
5
5
  out(text: string): void;
@@ -9,7 +9,7 @@
9
9
  // ISO 8601. The topic is a dotted logger name: the program, then the area the record
10
10
  // comes from (`ckan.cli`, `ckan.api`, `ckan.http`). stdout carries data
11
11
  // only and is not touched; neither is `--help`/`--version`.
12
- import { cutText, toWellFormed } from "../client/errors.js";
12
+ import { cutText, toWellFormed } from "@maschinenlesbar.org/ckan-client/internal/errors";
13
13
  /** The log formats `--log-format` takes. */
14
14
  export const LOG_FORMATS = ["text", "jsonl"];
15
15
  /** The format without `--log-format`. */
@@ -5,10 +5,10 @@ import { readFileSync } from "node:fs";
5
5
  import { fileURLToPath } from "node:url";
6
6
  import { Command, InvalidArgumentError, Option } from "commander";
7
7
  import { defaultIO } from "./io.js";
8
- import { CkanClient } from "../client/client.js";
9
- import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "../client/engine.js";
10
- import { MAX_TIMEOUT_MS } from "../client/http.js";
11
- import { cutForMessage, redactUrl } from "../client/errors.js";
8
+ import { CkanClient } from "@maschinenlesbar.org/ckan-client/internal/client";
9
+ import { DEFAULT_BASE_URL, MAX_RETRIES, MAX_RETRY_AFTER_MS } from "@maschinenlesbar.org/ckan-client/internal/engine";
10
+ import { MAX_TIMEOUT_MS } from "@maschinenlesbar.org/ckan-client/internal/http";
11
+ import { cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
12
12
  import { once, parseBaseUrl, parseBoundedInt, parseHeaderValue, parseIntArg, parsePortal } from "./shared.js";
13
13
  import { registerCatalogueCommands } from "./commands/catalogue.js";
14
14
  import { registerPortalCommands } from "./commands/portal.js";
@@ -5,7 +5,7 @@ import { CommanderError } from "commander";
5
5
  import { buildProgram, defaultDeps } from "./program.js";
6
6
  import { logOf } from "./io.js";
7
7
  import { createLogger, logFormatFromArgv } from "./log.js";
8
- import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "../client/errors.js";
8
+ import { CkanActionError, CkanApiError, CkanError, CkanNetworkError, CkanParseError, CkanValidationError, credentialsIn, echoedCredentialForms, redactCredentials, redactSecrets, } from "@maschinenlesbar.org/ckan-client/internal/errors";
9
9
  /** The exit code of a usage error: commander's own for a rejected option value. */
10
10
  const USAGE_EXIT = 1;
11
11
  /**
@@ -1,5 +1,5 @@
1
1
  import { type CliDeps } from "./io.js";
2
- import { type EngineOptions } from "../client/engine.js";
2
+ import { type EngineOptions, type RetryEvent } from "@maschinenlesbar.org/ckan-client/internal/engine";
3
3
  /** commander value-parser: a non-negative integer. */
4
4
  export declare function parseIntArg(value: string): number;
5
5
  /**
@@ -74,6 +74,8 @@ export declare function renderJson(deps: CliDeps, global: GlobalOptions, value:
74
74
  * stdout and the exit code are untouched.
75
75
  */
76
76
  export declare function warnOnCleartext(deps: CliDeps, global: GlobalOptions): void;
77
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
78
+ export declare function retryMessage(event: RetryEvent): string;
77
79
  export interface ActionContext {
78
80
  client: ReturnType<CliDeps["createClient"]>;
79
81
  global: GlobalOptions;
@@ -2,11 +2,11 @@
2
2
  // option resolver, and the JSON result renderer.
3
3
  import { InvalidArgumentError } from "commander";
4
4
  import { logOf } from "./io.js";
5
- import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "../client/engine.js";
6
- import { CkanError, cutForMessage, redactUrl } from "../client/errors.js";
7
- import { findPortal } from "../client/portals.js";
8
- import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "../client/validate.js";
9
- import { PORTALS } from "../client/portals-list.js";
5
+ import { cleartextProblem, DEFAULT_BASE_URL, isBidiControl } from "@maschinenlesbar.org/ckan-client/internal/engine";
6
+ import { CkanError, cutForMessage, redactUrl } from "@maschinenlesbar.org/ckan-client/internal/errors";
7
+ import { findPortal } from "@maschinenlesbar.org/ckan-client/internal/portals";
8
+ import { baseUrlProblem, blankProblem, countProblem, headerValueProblem, intRangeProblem } from "@maschinenlesbar.org/ckan-client/internal/validate";
9
+ import { PORTALS } from "@maschinenlesbar.org/ckan-client/internal/portals-list";
10
10
  /** commander value-parser: a non-negative integer. */
11
11
  export function parseIntArg(value) {
12
12
  // Validate the literal shape rather than trusting Number(): that rejects
@@ -182,6 +182,19 @@ export function warnOnCleartext(deps, global) {
182
182
  if (problem !== undefined)
183
183
  logOf(deps).warn("http", problem);
184
184
  }
185
+ /** `HTTP 503 from host: retry 1 of 3 in 2 s` (host only; whole seconds, ms under 1 s). */
186
+ export function retryMessage(event) {
187
+ let host;
188
+ try {
189
+ host = new URL(event.url).host;
190
+ }
191
+ catch {
192
+ host = "the server";
193
+ }
194
+ const why = event.status === undefined ? "connection reset" : `HTTP ${event.status}`;
195
+ const wait = event.delayMs < 1000 ? `${event.delayMs} ms` : `${Math.round(event.delayMs / 1000)} s`;
196
+ return `${why} from ${host}: retry ${event.retry} of ${event.maxRetries} in ${wait}`;
197
+ }
185
198
  /**
186
199
  * Wrap an async command action with consistent global-option resolution and
187
200
  * client construction. The callback receives a context (client + resolved global
@@ -196,7 +209,9 @@ export function action(deps, fn) {
196
209
  const positionals = args.slice(0, Math.max(0, args.length - 2));
197
210
  const global = command.optsWithGlobals();
198
211
  warnOnCleartext(deps, global);
199
- const client = deps.createClient(toEngineOptions(global));
212
+ const options = toEngineOptions(global);
213
+ options.onRetry = (event) => logOf(deps).warn("http", retryMessage(event));
214
+ const client = deps.createClient(options);
200
215
  await fn({ client, global, opts: command.opts() }, positionals);
201
216
  };
202
217
  }
@@ -1 +1 @@
1
- export * from "./client/index.js";
1
+ export * from "@maschinenlesbar.org/ckan-client";
package/dist/src/index.js CHANGED
@@ -1,2 +1,3 @@
1
1
  // Library root. Import from here when using ckan-cli as a dependency.
2
- export * from "./client/index.js";
2
+ // The client lives in the workspace package packages/client, bundled into this package.
3
+ export * from "@maschinenlesbar.org/ckan-client";
@@ -0,0 +1,22 @@
1
+ (The MIT License)
2
+
3
+ Copyright (c) 2011 TJ Holowaychuk <tj@vision-media.ca>
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ 'Software'), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19
+ IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
20
+ CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
21
+ TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
22
+ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.