@maschinenlesbar.org/ckan-cli 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -4
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +3 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +7 -5
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.d.ts +10 -10
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/engine.js +69 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +1 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +1 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +6 -6
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/types.d.ts +11 -11
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.d.ts +11 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/validate.js +13 -0
- package/dist/src/cli/shared.js +31 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -277,10 +277,12 @@ CKAN's error text is shown on stderr (in an `ERROR` record of `ckan.api`), e.g.
|
|
|
277
277
|
`Bad request - Action name not known: …`.
|
|
278
278
|
|
|
279
279
|
> **Numbers pass through JavaScript.** The output is the parsed and re-serialised
|
|
280
|
-
> JSON
|
|
281
|
-
>
|
|
282
|
-
>
|
|
283
|
-
>
|
|
280
|
+
> JSON. A number that would not print as the server wrote it (an integer above 2⁵³,
|
|
281
|
+
> 9,007,199,254,740,992, that loses its last digits, a number beyond the double range,
|
|
282
|
+
> `1e400`, or one too small, `1e-400`) is an error (exit 1, `the number … cannot be
|
|
283
|
+
> represented exactly`), never a changed value; one the server wrote in a double's
|
|
284
|
+
> shortest form (`1E23`) prints as written. CKAN metadata rarely holds such numbers; for
|
|
285
|
+
> an exact copy of the raw answer, use `curl` on `<site>/api/3/action/<name>`.
|
|
284
286
|
|
|
285
287
|
## Global options
|
|
286
288
|
|
|
@@ -44,7 +44,8 @@ export declare class CkanClient {
|
|
|
44
44
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
45
45
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
46
46
|
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
47
|
-
* refused (CkanValidationError), never sent or dropped.
|
|
47
|
+
* refused (CkanValidationError), never sent or dropped. The search text `q` goes
|
|
48
|
+
* out trimmed and in NFC (`normalizeText`); filters, sort and facet fields as given.
|
|
48
49
|
*/
|
|
49
50
|
packageSearch(params?: PackageSearchParams): Promise<PackageSearchResult>;
|
|
50
51
|
/** A single dataset by id or name. */
|
|
@@ -64,7 +65,7 @@ export declare class CkanClient {
|
|
|
64
65
|
organizationList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
65
66
|
/** Group names, or full objects with `all_fields` (paged like organizationList). */
|
|
66
67
|
groupList(params?: GroupListParams): Promise<JsonValue[]>;
|
|
67
|
-
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
68
|
+
/** Tags, optionally only those containing a substring (a blank one is refused; sent trimmed, in NFC). */
|
|
68
69
|
tagList(params?: TagListParams): Promise<string[]>;
|
|
69
70
|
/** The licences this portal offers. */
|
|
70
71
|
licenseList(): Promise<License[]>;
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
// Action API (`<site>/api/3/action`).
|
|
3
3
|
import { DEFAULT_BASE_URL, RequestEngine, sanitizeServerText, validateBaseUrl } from "./engine.js";
|
|
4
4
|
import { CkanActionError, CkanParseError, CkanValidationError, cutForMessage, describeCkanError, redactUrl } from "./errors.js";
|
|
5
|
-
import { assertValid, blankProblem, countProblem, facetLimitProblem, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
5
|
+
import { assertValid, blankProblem, countProblem, facetLimitProblem, normalizeText, queryValueProblem, textListProblem, textProblem, } from "./validate.js";
|
|
6
6
|
const ACTION = "/api/3/action";
|
|
7
7
|
/**
|
|
8
8
|
* CKAN action names are always `[a-z0-9_]+`. Restricting to that allowlist closes
|
|
@@ -197,7 +197,8 @@ export class CkanClient {
|
|
|
197
197
|
* match). `fq_list` is only used for two or more, because CKAN splits a lone
|
|
198
198
|
* `fq_list` value into characters. Facet fields go out as the JSON list CKAN
|
|
199
199
|
* expects in `facet.field`. A blank `q`, `sort`, filter or facet field is
|
|
200
|
-
* refused (CkanValidationError), never sent or dropped.
|
|
200
|
+
* refused (CkanValidationError), never sent or dropped. The search text `q` goes
|
|
201
|
+
* out trimmed and in NFC (`normalizeText`); filters, sort and facet fields as given.
|
|
201
202
|
*/
|
|
202
203
|
async packageSearch(params = {}) {
|
|
203
204
|
assertParams(params);
|
|
@@ -215,7 +216,7 @@ export class CkanClient {
|
|
|
215
216
|
const fq = params.fq ?? [];
|
|
216
217
|
const facetFields = params.facet_field ?? [];
|
|
217
218
|
return this.typed("package_search", {
|
|
218
|
-
q: params.q,
|
|
219
|
+
q: params.q === undefined ? undefined : normalizeText(params.q),
|
|
219
220
|
fq: fq.length === 1 ? fq[0] : undefined,
|
|
220
221
|
fq_list: fq.length > 1 ? fq : undefined,
|
|
221
222
|
rows: params.rows,
|
|
@@ -260,12 +261,13 @@ export class CkanClient {
|
|
|
260
261
|
groupList(params = {}) {
|
|
261
262
|
return this.groupOrOrgList("group_list", params);
|
|
262
263
|
}
|
|
263
|
-
/** Tags, optionally only those containing a substring (a blank one is refused). */
|
|
264
|
+
/** Tags, optionally only those containing a substring (a blank one is refused; sent trimmed, in NFC). */
|
|
264
265
|
async tagList(params = {}) {
|
|
265
266
|
assertParams(params);
|
|
266
267
|
assertKeys("tagList", "tag_list", params, TAG_LIST_KEYS);
|
|
267
268
|
assertText("query", params.query);
|
|
268
|
-
|
|
269
|
+
const query = params.query === undefined ? undefined : normalizeText(params.query);
|
|
270
|
+
return this.typed("tag_list", { query }, Array.isArray, "an array");
|
|
269
271
|
}
|
|
270
272
|
/** The licences this portal offers. */
|
|
271
273
|
licenseList() {
|
|
@@ -10,35 +10,35 @@ export interface RawResponse {
|
|
|
10
10
|
}
|
|
11
11
|
export interface EngineOptions {
|
|
12
12
|
/** Base URL of the API. Defaults to https://suche.transparenz.hamburg.de (the Hamburg Transparenzportal) */
|
|
13
|
-
baseUrl?: string;
|
|
13
|
+
baseUrl?: string | undefined;
|
|
14
14
|
/** Swappable transport. Defaults to the built-in node http/https transport. */
|
|
15
|
-
transport?: Transport;
|
|
15
|
+
transport?: Transport | undefined;
|
|
16
16
|
/**
|
|
17
17
|
* Value of the User-Agent header. Must be sendable (see `assertHeaderValue`):
|
|
18
18
|
* not blank, no control characters but tab, nothing above U+00FF.
|
|
19
19
|
*/
|
|
20
|
-
userAgent?: string;
|
|
20
|
+
userAgent?: string | undefined;
|
|
21
21
|
/**
|
|
22
22
|
* Per-request timeout in milliseconds, 0 to `MAX_TIMEOUT_MS` (2^31 - 1 ms); 0 disables.
|
|
23
23
|
* It covers the whole response, body included, and the engine enforces it for every
|
|
24
24
|
* transport (the request also gets an AbortSignal that fires at the deadline).
|
|
25
25
|
*/
|
|
26
|
-
timeoutMs?: number;
|
|
26
|
+
timeoutMs?: number | undefined;
|
|
27
27
|
/**
|
|
28
28
|
* Number of automatic retries for transient (429/503) responses; a reset or other
|
|
29
29
|
* network error is not retried. Each waits `retryDelayMs * attempt`, or the
|
|
30
30
|
* response's `Retry-After` when that is longer (up to `MAX_RETRY_AFTER_MS`; a longer
|
|
31
31
|
* one is not retried, and the CkanApiError says so). 0 to `MAX_RETRIES`; defaults to 2.
|
|
32
32
|
*/
|
|
33
|
-
maxRetries?: number;
|
|
33
|
+
maxRetries?: number | undefined;
|
|
34
34
|
/**
|
|
35
35
|
* Base backoff between retries in milliseconds (grows linearly), 0 to
|
|
36
36
|
* `MAX_RETRY_AFTER_MS` (30 000); defaults to 200. It is also the floor under a
|
|
37
37
|
* `Retry-After`: the header can lengthen a wait, never shorten it.
|
|
38
38
|
*/
|
|
39
|
-
retryDelayMs?: number;
|
|
39
|
+
retryDelayMs?: number | undefined;
|
|
40
40
|
/** Number of HTTP redirects (301/302/303/307/308) to follow, 0 to `MAX_REDIRECTS`. Defaults to 5. */
|
|
41
|
-
maxRedirects?: number;
|
|
41
|
+
maxRedirects?: number | undefined;
|
|
42
42
|
/**
|
|
43
43
|
* Hard cap on response body size in bytes (defends against memory exhaustion
|
|
44
44
|
* from a hostile/buggy endpoint). Defaults to 100 MiB; set to 0 for no limit.
|
|
@@ -49,14 +49,14 @@ export interface EngineOptions {
|
|
|
49
49
|
* the constructor throws CkanValidationError otherwise (a negative or NaN
|
|
50
50
|
* timeout or cap would silently switch that guard off).
|
|
51
51
|
*/
|
|
52
|
-
maxResponseBytes?: number;
|
|
52
|
+
maxResponseBytes?: number | undefined;
|
|
53
53
|
/** Injectable sleep, primarily for deterministic tests. */
|
|
54
|
-
sleep?: (ms: number) => Promise<void
|
|
54
|
+
sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
55
55
|
/**
|
|
56
56
|
* Called once per retry, right before the backoff sleep, for each retried 429/503
|
|
57
57
|
* and reset connection; never when there is no retry. A throw is swallowed.
|
|
58
58
|
*/
|
|
59
|
-
onRetry?: (event: RetryEvent) => void;
|
|
59
|
+
onRetry?: ((event: RetryEvent) => void) | undefined;
|
|
60
60
|
}
|
|
61
61
|
/** What `EngineOptions.onRetry` is told about one retry. */
|
|
62
62
|
export interface RetryEvent {
|
|
@@ -513,8 +513,9 @@ export class RequestEngine {
|
|
|
513
513
|
async getJson(path, query) {
|
|
514
514
|
const res = await this.request("GET", path, { query, accept: "application/json" });
|
|
515
515
|
const text = decodeBody(res.data, res.contentType, res.url);
|
|
516
|
+
let parsed;
|
|
516
517
|
try {
|
|
517
|
-
|
|
518
|
+
parsed = JSON.parse(text);
|
|
518
519
|
}
|
|
519
520
|
catch (cause) {
|
|
520
521
|
// Name the URL and what came back: a site that is not a CKAN (or not at this
|
|
@@ -527,6 +528,13 @@ export class RequestEngine {
|
|
|
527
528
|
: `Expected JSON from ${cutForMessage(redactUrl(res.url))} but got ${mediaType || "a body that is not JSON"}`;
|
|
528
529
|
throw new CkanParseError(message, { cause: this.scrubCause(cause) });
|
|
529
530
|
}
|
|
531
|
+
// JSON.parse rounds and overflows numbers silently: refuse the answer rather than print a
|
|
532
|
+
// changed value.
|
|
533
|
+
const inexact = firstInexactNumber(text);
|
|
534
|
+
if (inexact !== undefined) {
|
|
535
|
+
throw new CkanParseError(`Unexpected response from ${cutForMessage(redactUrl(res.url))}: the number ${cutForMessage(inexact)} cannot be represented exactly.`);
|
|
536
|
+
}
|
|
537
|
+
return parsed;
|
|
530
538
|
}
|
|
531
539
|
toApiError(method, url, status, body, hint, retry) {
|
|
532
540
|
const text = this.scrub(body.toString("utf8"));
|
|
@@ -627,3 +635,63 @@ function resolveLocation(location, base) {
|
|
|
627
635
|
return undefined;
|
|
628
636
|
}
|
|
629
637
|
}
|
|
638
|
+
/** A JSON number token, matched at a given position (sticky). */
|
|
639
|
+
const JSON_NUMBER = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/y;
|
|
640
|
+
/**
|
|
641
|
+
* Whether the double `value` (finite) prints as the number `literal` wrote, checked where a
|
|
642
|
+
* double can be off by a whole number or more: past 2^53, where every double is an integer, and
|
|
643
|
+
* at 0, which a literal too small for a double becomes. Past 2^53 the CLI prints `String(value)`,
|
|
644
|
+
* the shortest decimal that reads back as the same double: `12345678901234567890` prints as
|
|
645
|
+
* `12345678901234567000`, a changed value, while `1E23` or `1.7976931348623157E308` (a double a
|
|
646
|
+
* server printed in its own shortest form) print as the same number.
|
|
647
|
+
*/
|
|
648
|
+
function exactAtTheEdges(literal, value) {
|
|
649
|
+
if (value !== 0 && Math.abs(value) <= Number.MAX_SAFE_INTEGER)
|
|
650
|
+
return true;
|
|
651
|
+
const written = decimalOf(literal);
|
|
652
|
+
if (value === 0)
|
|
653
|
+
return written.digits === "";
|
|
654
|
+
const printed = decimalOf(String(value));
|
|
655
|
+
return written.negative === printed.negative && written.digits === printed.digits && written.exponent === printed.exponent;
|
|
656
|
+
}
|
|
657
|
+
/** A decimal literal as sign, significant digits (no leading or trailing zeros) and exponent: digits × 10^exponent. */
|
|
658
|
+
function decimalOf(text) {
|
|
659
|
+
const [, sign = "", whole = "", fraction = "", exp = "0"] = /^(-?)(\d+)(?:\.(\d+))?(?:[eE]([+-]?\d+))?$/.exec(text) ?? [];
|
|
660
|
+
const all = (whole + fraction).replace(/^0+/, "");
|
|
661
|
+
const digits = all.replace(/0+$/, "");
|
|
662
|
+
return { negative: sign === "-", digits, exponent: Number(exp) - fraction.length + (all.length - digits.length) };
|
|
663
|
+
}
|
|
664
|
+
/**
|
|
665
|
+
* The first number literal in `json` (JSON that parsed) that JSON.parse cannot carry exactly, or
|
|
666
|
+
* undefined: one out of a double's range (`1e400` became Infinity, printed as `null`), one past
|
|
667
|
+
* 2^53 that it rounds, in any form (`12345678901234567890`, `12345678901234567890.0`,
|
|
668
|
+
* `9007199254740993e0`, `9007199254740992.5`), and a non-zero one too small for a double
|
|
669
|
+
* (`1e-400` became 0). Below 2^53 a decimal fraction keeps a double's usual precision, as JSON
|
|
670
|
+
* means it. A linear scan that skips strings; a reviver would recurse per nesting level.
|
|
671
|
+
*/
|
|
672
|
+
function firstInexactNumber(json) {
|
|
673
|
+
for (let i = 0; i < json.length; i++) {
|
|
674
|
+
const c = json.charCodeAt(i);
|
|
675
|
+
if (c === 0x22) {
|
|
676
|
+
// a string: skip to its closing quote, past escapes
|
|
677
|
+
for (i++; i < json.length; i++) {
|
|
678
|
+
const d = json.charCodeAt(i);
|
|
679
|
+
if (d === 0x5c)
|
|
680
|
+
i++;
|
|
681
|
+
else if (d === 0x22)
|
|
682
|
+
break;
|
|
683
|
+
}
|
|
684
|
+
}
|
|
685
|
+
else if (c === 0x2d || (c >= 0x30 && c <= 0x39)) {
|
|
686
|
+
JSON_NUMBER.lastIndex = i;
|
|
687
|
+
const literal = JSON_NUMBER.exec(json)?.[0];
|
|
688
|
+
if (literal === undefined)
|
|
689
|
+
continue;
|
|
690
|
+
const value = Number(literal);
|
|
691
|
+
if (!Number.isFinite(value) || !exactAtTheEdges(literal, value))
|
|
692
|
+
return literal;
|
|
693
|
+
i += literal.length - 1;
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
return undefined;
|
|
697
|
+
}
|
|
@@ -6,7 +6,7 @@ export type { Transport, HttpRequest, HttpResponse } from "./http.js";
|
|
|
6
6
|
export { buildQueryString } from "./query.js";
|
|
7
7
|
export type { QueryParams, QueryValue } from "./query.js";
|
|
8
8
|
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
9
|
-
export { assertValid } from "./validate.js";
|
|
9
|
+
export { assertValid, normalizeText } from "./validate.js";
|
|
10
10
|
export type { Problem } from "./validate.js";
|
|
11
11
|
export { PORTALS } from "./portals-list.js";
|
|
12
12
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
@@ -4,7 +4,7 @@ export { RequestEngine, DEFAULT_BASE_URL, assertHeaderValue, MAX_REDIRECTS, MAX_
|
|
|
4
4
|
export { MAX_TIMEOUT_MS, nodeHttpTransport } from "./http.js";
|
|
5
5
|
export { buildQueryString } from "./query.js";
|
|
6
6
|
export { CkanError, CkanApiError, CkanActionError, CkanNetworkError, CkanParseError, CkanValidationError, describeCkanError, redactUrl, credentialsIn, redactCredentials, cutForMessage, MAX_MESSAGE_VALUE_LENGTH, } from "./errors.js";
|
|
7
|
-
export { assertValid } from "./validate.js";
|
|
7
|
+
export { assertValid, normalizeText } from "./validate.js";
|
|
8
8
|
export { PORTALS } from "./portals-list.js";
|
|
9
9
|
export { DEFAULT_CHECK_CONCURRENCY, checkPortal, checkPortalUrls, checkPortals, findPortal, mapLimit, portalKey, withCheck, } from "./portals.js";
|
|
10
10
|
export * from "./types.js";
|
|
@@ -40,18 +40,18 @@ export declare const DEFAULT_CHECK_CONCURRENCY = 6;
|
|
|
40
40
|
/** Options for `checkPortalUrls` and `checkPortals`. */
|
|
41
41
|
export interface CheckPortalsOptions {
|
|
42
42
|
/** Client options for every check; `baseUrl` is replaced by each portal's URL. */
|
|
43
|
-
engineOptions?: EngineOptions;
|
|
43
|
+
engineOptions?: EngineOptions | undefined;
|
|
44
44
|
/** Builds the client for one portal. Defaults to `new CkanClient(options)`. */
|
|
45
|
-
createClient?: (options: EngineOptions) => CkanClient;
|
|
45
|
+
createClient?: ((options: EngineOptions) => CkanClient) | undefined;
|
|
46
46
|
/** Checks in flight at once, 1 or more. Defaults to `DEFAULT_CHECK_CONCURRENCY` (6). */
|
|
47
|
-
concurrency?: number;
|
|
47
|
+
concurrency?: number | undefined;
|
|
48
48
|
/**
|
|
49
49
|
* When set, a failed check is repeated once after this pause (ms), so one
|
|
50
50
|
* timeout does not count as down. Unset (the default): a single try.
|
|
51
51
|
*/
|
|
52
|
-
retryDelayMs?: number;
|
|
52
|
+
retryDelayMs?: number | undefined;
|
|
53
53
|
/** Injectable sleep for the retry pause, for tests. */
|
|
54
|
-
sleep?: (ms: number) => Promise<void
|
|
54
|
+
sleep?: ((ms: number) => Promise<void>) | undefined;
|
|
55
55
|
}
|
|
56
56
|
/**
|
|
57
57
|
* Check the CKAN portals at `urls` live (`checkPortal`), with at most
|
|
@@ -66,5 +66,5 @@ export declare function checkPortalUrls(urls: readonly string[], options?: Check
|
|
|
66
66
|
* What `ckan portals --check` prints.
|
|
67
67
|
*/
|
|
68
68
|
export declare function checkPortals(portals?: readonly Portal[], options?: CheckPortalsOptions & {
|
|
69
|
-
date?: string;
|
|
69
|
+
date?: string | undefined;
|
|
70
70
|
}): Promise<Portal[]>;
|
|
@@ -35,22 +35,22 @@ export interface PackageSearchResult {
|
|
|
35
35
|
/** Parameters for `package_search`. */
|
|
36
36
|
export interface PackageSearchParams {
|
|
37
37
|
/** Solr query string, e.g. `elbe` or `title:Haushalt`. */
|
|
38
|
-
q?: string;
|
|
38
|
+
q?: string | undefined;
|
|
39
39
|
/** Filter queries, e.g. `["organization:allris"]`. All must match. */
|
|
40
|
-
fq?: string[];
|
|
40
|
+
fq?: string[] | undefined;
|
|
41
41
|
/** Page size. CKAN caps it server-side (`ckan.search.rows_max`, 1000 by default). */
|
|
42
|
-
rows?: number;
|
|
42
|
+
rows?: number | undefined;
|
|
43
43
|
/** Offset of the first result, for paging. */
|
|
44
|
-
start?: number;
|
|
44
|
+
start?: number | undefined;
|
|
45
45
|
/** e.g. `"metadata_modified desc"`. */
|
|
46
|
-
sort?: string;
|
|
46
|
+
sort?: string | undefined;
|
|
47
47
|
/**
|
|
48
48
|
* Facet fields to count, e.g. `["organization", "res_format"]`. Counts come
|
|
49
49
|
* back in `search_facets`.
|
|
50
50
|
*/
|
|
51
|
-
facet_field?: string[];
|
|
51
|
+
facet_field?: string[] | undefined;
|
|
52
52
|
/** Maximum number of values returned per facet (CKAN's default is 50; -1 = all). */
|
|
53
|
-
facet_limit?: number;
|
|
53
|
+
facet_limit?: number | undefined;
|
|
54
54
|
}
|
|
55
55
|
/** An organization (a data publisher) or a group (a theme/category). */
|
|
56
56
|
export type Organization = JsonObject;
|
|
@@ -64,18 +64,18 @@ export interface ListParams {
|
|
|
64
64
|
* is refused: CKAN reads `limit=0` as "no limit", which the `all_fields` pager
|
|
65
65
|
* would read as "no entries".
|
|
66
66
|
*/
|
|
67
|
-
limit?: number;
|
|
68
|
-
offset?: number;
|
|
67
|
+
limit?: number | undefined;
|
|
68
|
+
offset?: number | undefined;
|
|
69
69
|
}
|
|
70
70
|
/** Parameters for `organization_list` / `group_list`. */
|
|
71
71
|
export interface GroupListParams extends ListParams {
|
|
72
72
|
/** Return full objects instead of just names. */
|
|
73
|
-
all_fields?: boolean;
|
|
73
|
+
all_fields?: boolean | undefined;
|
|
74
74
|
}
|
|
75
75
|
/** Parameters for `tag_list`. */
|
|
76
76
|
export interface TagListParams {
|
|
77
77
|
/** Only tags containing this substring. */
|
|
78
|
-
query?: string;
|
|
78
|
+
query?: string | undefined;
|
|
79
79
|
}
|
|
80
80
|
/** A licence a portal offers (`license_list`). */
|
|
81
81
|
export type License = JsonObject;
|
|
@@ -17,6 +17,17 @@ export declare function isBlank(value: string): boolean;
|
|
|
17
17
|
export declare function blankProblem(value: unknown): string | undefined;
|
|
18
18
|
/** A non-blank string (an id, a query, a sort expression); anything else is refused. */
|
|
19
19
|
export declare function textProblem(value: unknown): string | undefined;
|
|
20
|
+
/**
|
|
21
|
+
* A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
|
|
22
|
+
* with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
|
|
23
|
+
* "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
|
|
24
|
+
* composed form matches). Inner whitespace is kept. Used for the search text (`q`) and
|
|
25
|
+
* the tag query, never for ids, Solr filters (`fq`), sort expressions or field names.
|
|
26
|
+
* NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
|
|
27
|
+
* meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
|
|
28
|
+
* forms) that a phrase or a wildcard may mean literally.
|
|
29
|
+
*/
|
|
30
|
+
export declare function normalizeText(value: string): string;
|
|
20
31
|
/** A list of non-blank strings (filter queries, facet fields). */
|
|
21
32
|
export declare function textListProblem(value: unknown): string | undefined;
|
|
22
33
|
/**
|
|
@@ -36,6 +36,19 @@ export function textProblem(value) {
|
|
|
36
36
|
return "Expected a string.";
|
|
37
37
|
return blankProblem(value);
|
|
38
38
|
}
|
|
39
|
+
/**
|
|
40
|
+
* A free-text value as CKAN matches it: without surrounding whitespace (a value pasted
|
|
41
|
+
* with a trailing space matched nothing) and in composed Unicode, NFC (a decomposed
|
|
42
|
+
* "Gewässer", a plus U+0308 as pasted from a macOS file name, matched nothing the
|
|
43
|
+
* composed form matches). Inner whitespace is kept. Used for the search text (`q`) and
|
|
44
|
+
* the tag query, never for ids, Solr filters (`fq`), sort expressions or field names.
|
|
45
|
+
* NFC changes no character of Solr's query syntax (all ASCII), so a `q` keeps its
|
|
46
|
+
* meaning; NFKC is not used, because it would rewrite characters (`fi` to `fi`, full-width
|
|
47
|
+
* forms) that a phrase or a wildcard may mean literally.
|
|
48
|
+
*/
|
|
49
|
+
export function normalizeText(value) {
|
|
50
|
+
return value.trim().normalize("NFC");
|
|
51
|
+
}
|
|
39
52
|
/** A list of non-blank strings (filter queries, facet fields). */
|
|
40
53
|
export function textListProblem(value) {
|
|
41
54
|
if (!Array.isArray(value))
|
package/dist/src/cli/shared.js
CHANGED
|
@@ -151,10 +151,18 @@ export function escapeControlChars(json) {
|
|
|
151
151
|
* which is why the message suggests --compact. The RangeError becomes a CkanError so
|
|
152
152
|
* the CLI prints a clear message instead of "Unexpected error: Maximum call stack
|
|
153
153
|
* size exceeded".
|
|
154
|
+
*
|
|
155
|
+
* Compact output takes V8's fast path (no gap) unless the answer holds a huge string: on
|
|
156
|
+
* Node 26 the fast path aborts the process ("Zone Allocation failed", exit 134, a native
|
|
157
|
+
* dump on stderr) on a string of some 250 MB that JSON.parse made, while the gap path (an
|
|
158
|
+
* empty gap, the same text) prints it. The fast path stays the default because on Node 26
|
|
159
|
+
* only it is iterative and prints nesting the gap path refuses (~7,000 levels).
|
|
154
160
|
*/
|
|
155
161
|
function stringifyJson(value, compact) {
|
|
156
162
|
try {
|
|
157
|
-
|
|
163
|
+
if (!compact)
|
|
164
|
+
return JSON.stringify(value, null, 2);
|
|
165
|
+
return holdsHugeString(value) ? JSON.stringify(value, null, "") : JSON.stringify(value);
|
|
158
166
|
}
|
|
159
167
|
catch (err) {
|
|
160
168
|
if (err instanceof RangeError) {
|
|
@@ -165,6 +173,28 @@ function stringifyJson(value, compact) {
|
|
|
165
173
|
throw err;
|
|
166
174
|
}
|
|
167
175
|
}
|
|
176
|
+
/** A string or key this long sends --compact through the gap path: a quarter of where Node 26's fast path aborts. */
|
|
177
|
+
const HUGE_STRING_LENGTH = 64 * 1024 * 1024;
|
|
178
|
+
/** Whether `value` holds a string or key of HUGE_STRING_LENGTH or more. Iterative, so nesting costs no stack. */
|
|
179
|
+
function holdsHugeString(value) {
|
|
180
|
+
const stack = [value];
|
|
181
|
+
while (stack.length > 0) {
|
|
182
|
+
const item = stack.pop();
|
|
183
|
+
if (typeof item === "string") {
|
|
184
|
+
if (item.length >= HUGE_STRING_LENGTH)
|
|
185
|
+
return true;
|
|
186
|
+
}
|
|
187
|
+
else if (typeof item === "object" && item !== null) {
|
|
188
|
+
for (const [key, element] of Object.entries(item)) {
|
|
189
|
+
if (key.length >= HUGE_STRING_LENGTH)
|
|
190
|
+
return true;
|
|
191
|
+
if (typeof element === "string" || (typeof element === "object" && element !== null))
|
|
192
|
+
stack.push(element);
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
return false;
|
|
197
|
+
}
|
|
168
198
|
/** Render a JSON value to stdout, pretty by default, compact with --compact. */
|
|
169
199
|
export function renderJson(deps, global, value) {
|
|
170
200
|
const text = escapeControlChars(stringifyJson(value, global.compact === true));
|
package/package.json
CHANGED