@maschinenlesbar.org/ckan-cli 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. package/README.md +56 -0
  2. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +25 -1
  3. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +63 -22
  4. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -1
  5. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -1
  6. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.d.ts +441 -0
  7. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.js +1211 -0
  8. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +2 -0
  9. package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +2 -2
  10. package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.d.ts +1 -0
  11. package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.js +1 -0
  12. package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.d.ts +80 -0
  13. package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.js +124 -0
  14. package/dist/node_modules/@maschinenlesbar.org/ckan-log/package.json +25 -0
  15. package/dist/src/cli/index.js +3 -1
  16. package/dist/src/cli/io.d.ts +17 -0
  17. package/dist/src/cli/io.js +14 -0
  18. package/dist/src/cli/limits.d.ts +19 -0
  19. package/dist/src/cli/limits.js +53 -0
  20. package/dist/src/cli/program.js +3 -0
  21. package/dist/src/cli/run.js +9 -1
  22. package/dist/src/cli/shared.d.ts +4 -0
  23. package/dist/src/cli/shared.js +16 -1
  24. package/node_modules/commander/Readme.md +6 -32
  25. package/node_modules/commander/index.js +1 -1
  26. package/node_modules/commander/lib/command.js +22 -9
  27. package/node_modules/commander/lib/help.js +6 -2
  28. package/node_modules/commander/lib/option.js +1 -1
  29. package/node_modules/commander/package.json +2 -1
  30. package/node_modules/commander/typings/index.d.ts +6 -3
  31. package/package.json +2 -2
@@ -0,0 +1,441 @@
1
+ import { type CkanClient } from "./client.js";
2
+ import type { JsonValue, Portal } from "./types.js";
3
+ /** A portal of the built-in list by its id (any case); a URL or an unknown id is refused. */
4
+ export declare function portalById(id: string): Portal;
5
+ /** The portal as an answer names it. */
6
+ export interface PortalRef {
7
+ id?: string;
8
+ title?: string;
9
+ url?: string;
10
+ }
11
+ /**
12
+ * The requests the operation that returned `answer` made, in order, each as the client sends it:
13
+ * under the client's own Action API (`package_search?q=…`), or as an absolute URL on another
14
+ * portal of the list. What its `source` names. Empty for an object no operation returned.
15
+ */
16
+ export declare function requestsOf(answer: object): readonly string[];
17
+ /** Folded for comparing names: lower case, no accents (ü as u, like CKAN's slugs), ß as ss. */
18
+ export declare function foldName(text: string): string;
19
+ /** The EU file-type vocabulary, the spelling of most format values on GovData and daten.bw. */
20
+ export declare const EU_FILE_TYPE = "http://publications.europa.eu/resource/authority/file-type/";
21
+ /** Formats that are services, not files. */
22
+ export declare const SERVICE_FORMATS: ReadonlySet<string>;
23
+ /**
24
+ * A format value as one short name: the EU URI's tail (`…/file-type/CSV` is CSV), upper case (csv
25
+ * and CSV are one), the EU `_SRVC` suffix dropped (WMS_SRVC is WMS), free-text and MIME spellings
26
+ * mapped (Shapefiles is SHP, text/csv is CSV). Undefined for a value that names no clear format:
27
+ * another vocabulary's URI, an unknown MIME type, a combined value ("Shape, CSV"), a word like
28
+ * "Download".
29
+ */
30
+ export declare function formatOf(raw: unknown): string | undefined;
31
+ /**
32
+ * The spellings portals store a format under: upper case (NRW, München), lower case (Hamburg; Berlin
33
+ * has both as separate values, and its filter is case-sensitive), the EU URI (GovData, daten.bw),
34
+ * free-text and MIME aliases.
35
+ */
36
+ export declare function formatSpellings(format: string): string[];
37
+ /** The Solr filter for datasets with a file in `format`, in any of its spellings. */
38
+ export declare function formatFilter(format: string): string;
39
+ export interface Licence {
40
+ /** As delivered (an id such as dl-de-by-2.0, or a URI). */
41
+ id: string;
42
+ /** A short name: "DL-DE BY 2.0", "CC BY 4.0"; the portal's title or the id when it is not known. */
43
+ label: string;
44
+ /** Whether it is an open licence; absent when the licence is not known. */
45
+ open?: boolean;
46
+ }
47
+ /**
48
+ * A licence with a short name; undefined for an empty value. The spellings of one licence across
49
+ * portals (dl-de-by-2.0, dl-by-de/2.0, the DCAT-AP.de URI, 2_0) get one label; `title` (the
50
+ * portal's license_title) names an unknown one.
51
+ */
52
+ export declare function licenceOf(raw: unknown, title?: unknown): Licence | undefined;
53
+ /**
54
+ * What a resource's link is: `file` (a file name at the end), `generated` (a file built on request:
55
+ * an export or download endpoint, a WFS GetFeature), `service` (a map or feature service, an OGC
56
+ * API, a feed: not a file), `page` (a landing page), `api` (another API call), `template` (a
57
+ * placeholder such as {yyyymm} or _yyyy. to fill in), `broken` (cut off by a harvester).
58
+ */
59
+ export type LinkKind = "file" | "generated" | "service" | "page" | "api" | "template" | "broken";
60
+ /** The extension at the end of a link's path (`.csv.gz` is CSV), upper case, or undefined. */
61
+ export declare function extensionOf(url: string): string | undefined;
62
+ /** Whether the link carries a key or token in its query (as published). */
63
+ export declare function hasKeyInLink(url: string): boolean;
64
+ /** The kind of a resource's link (see LinkKind), from the link and its format. */
65
+ export declare function linkKind(url: string, format: string | undefined): LinkKind;
66
+ /** One file (resource) of a dataset. */
67
+ export interface FileEntry {
68
+ id?: string;
69
+ name?: string;
70
+ /** Folded (CSV), or as delivered when it names no clear format. */
71
+ format?: string;
72
+ url: string;
73
+ kind: LinkKind;
74
+ /** The file name's extension, when it differs from the format (an .xlsx labelled XLS). */
75
+ extension?: string;
76
+ /** Bytes, when the metadata say. */
77
+ size?: number;
78
+ /** The file's own licence, when it names one. */
79
+ licence?: Licence;
80
+ modified?: string;
81
+ /** The link carries someone's key or token in its query, as published. */
82
+ keyInLink?: true;
83
+ }
84
+ /**
85
+ * The Transparenzportal Hamburg's document types (Informationsgegenstände), by the value its search
86
+ * index holds: Solr stems the stored words, so `extras_registerobject_type:gutacht` finds the
87
+ * Gutachten, while the published values (vertraege_oeff_interesse) and the stored ones
88
+ * (vertraegeoeffinteresse) find nothing, and some full words over-match (verwaltungsvorschriften
89
+ * finds the Dienstanweisungen too).
90
+ */
91
+ export declare const DOCUMENT_TYPES: Readonly<{
92
+ beschluss: "Beschlüsse";
93
+ bauleitplan: "Bauleitpläne";
94
+ baugenehm: "Baugenehmigungen";
95
+ statist: "Statistiken";
96
+ geodat: "Geodaten";
97
+ senatpetitum: "Senatspetita";
98
+ messung: "Messungen";
99
+ senatmitteil: "Senatsmitteilungen";
100
+ andereveroffentlichungspflicht: "andere Veröffentlichungspflicht";
101
+ ohneveroffentlichungspflicht: "ohne Veröffentlichungspflicht";
102
+ vertrageoffinteress: "Verträge von öffentlichem Interesse";
103
+ gutacht: "Gutachten";
104
+ vergleichbar: "vergleichbare Informationen";
105
+ verwaltungsvorschrift: "Verwaltungsvorschriften";
106
+ dienstanweis: "Dienstanweisungen";
107
+ daseinsvorsorg: "Daseinsvorsorge";
108
+ verwaltungsbelang: "Verwaltungsbelange";
109
+ stadtischebeteil: "städtische Beteiligungen";
110
+ baumkatast: "Baumkataster";
111
+ subvention: "Subventionen";
112
+ }>;
113
+ export type DocumentType = keyof typeof DOCUMENT_TYPES;
114
+ /**
115
+ * A document type's index value from that value, its German name, Hamburg's stored value
116
+ * (vertraegeoeffinteresse) or its published one (vertraege_oeff_interesse); undefined if none fits.
117
+ */
118
+ export declare function documentType(value: string): DocumentType | undefined;
119
+ export type DatasetSort = "relevance" | "modified" | "created" | "title";
120
+ /** CKAN's record types: datasets; Hamburg also documents and apps. */
121
+ export declare const RECORD_TYPES: readonly ["dataset", "document", "app"];
122
+ export type RecordType = (typeof RECORD_TYPES)[number];
123
+ /** The date field a range filters on. */
124
+ export type DateField = "created" | "modified";
125
+ /** What the dataset tools filter by. */
126
+ export interface DatasetFilters {
127
+ /** A portal of the built-in list by its id; else the client's own. */
128
+ portal?: string | undefined;
129
+ /** Solr query: words, "phrases", AND/OR/NOT, field:value (title:Radverkehr). */
130
+ query?: string | undefined;
131
+ /** An organization (publisher), by slug or name. */
132
+ organization?: string | undefined;
133
+ /** A group (theme, category), by slug or name. */
134
+ group?: string | undefined;
135
+ /** A tag, exactly as the portal spells it. */
136
+ tag?: string | undefined;
137
+ /** Datasets with a file in this format, in any of its spellings (CSV, GEOJSON, SHP, …). */
138
+ format?: string | undefined;
139
+ /** dataset, document or app (Hamburg has all three). */
140
+ type?: RecordType | undefined;
141
+ /** Only datasets with at least one file (num_resources ≥ 1): metadata-only records left out. */
142
+ withFiles?: boolean | undefined;
143
+ /** The date field of from and to: created (when the record entered the portal, default) or modified. */
144
+ dateField?: DateField | undefined;
145
+ /** From this day on (YYYY-MM-DD). */
146
+ from?: string | undefined;
147
+ /** Up to and including this day (YYYY-MM-DD). */
148
+ to?: string | undefined;
149
+ /** Raw Solr filter queries, ANDed (each its own filter). */
150
+ filters?: readonly string[] | undefined;
151
+ }
152
+ /** What a filtered search was given, as the answer repeats it. */
153
+ export type Searched = Record<string, JsonValue>;
154
+ /** An organization or group of a portal: its slug, its title, how many datasets it has. */
155
+ export interface OrganizationEntry {
156
+ name: string;
157
+ title: string;
158
+ datasets: number;
159
+ }
160
+ /** Several organizations or groups fit a name: the candidates, never a guess. */
161
+ export interface Ambiguity {
162
+ field: string;
163
+ value: string;
164
+ candidates: OrganizationEntry[];
165
+ total: number;
166
+ }
167
+ /** What every answer has. */
168
+ interface AnswerBase {
169
+ status: "ok" | "empty" | "ambiguous";
170
+ /** Why the answer is empty or ambiguous, and what to try. */
171
+ reason?: string;
172
+ /** The portal asked (from the built-in list; empty for another the server was pointed at). */
173
+ portal: PortalRef;
174
+ searched: Searched;
175
+ notes: string[];
176
+ ambiguous?: Ambiguity;
177
+ }
178
+ /** The most results a page may start after: CKAN reads further, but nobody pages that far. */
179
+ export declare const MAX_OFFSET = 100000;
180
+ export declare const DEFAULT_DATASETS = 10;
181
+ export declare const MAX_DATASETS = 50;
182
+ export interface FindDatasetsParams extends DatasetFilters {
183
+ sort?: DatasetSort | undefined;
184
+ offset?: number | undefined;
185
+ limit?: number | undefined;
186
+ }
187
+ /** One dataset of a search, with what decides its reuse. */
188
+ export interface DatasetHit {
189
+ /** The slug, for getDataset. */
190
+ name: string;
191
+ title?: string;
192
+ /** dataset, document or app. */
193
+ type?: string;
194
+ organization?: {
195
+ name: string;
196
+ title?: string;
197
+ };
198
+ /** The office or person behind it (author, else maintainer). */
199
+ author?: string;
200
+ description?: string;
201
+ /** When the record last changed (metadata_modified). */
202
+ modified?: string;
203
+ /** When the record entered the portal (metadata_created). */
204
+ created?: string;
205
+ /** How many files (resources) it has; 0 is a metadata-only record. */
206
+ files: number;
207
+ /** The formats of its files, folded (CSV, not csv or the EU URI). */
208
+ formats: string[];
209
+ /** Its licence, and any other its files name. */
210
+ licences: Licence[];
211
+ /** The attribution text the publisher asks for, where the record carries one. */
212
+ attribution?: string;
213
+ /** Group slugs. */
214
+ groups: string[];
215
+ /** With a format filter: whether a file of it is there. */
216
+ hasFormat?: boolean;
217
+ }
218
+ export interface FindDatasetsAnswer extends AnswerBase {
219
+ total: number;
220
+ offset: number;
221
+ datasets: DatasetHit[];
222
+ nextOffset?: number;
223
+ }
224
+ /**
225
+ * Datasets for a topic and filters on one portal, a page at a time (`total` counts all matches:
226
+ * read it, never page to count). Each hit names its formats folded, its licences and the
227
+ * attribution text where the record carries one. The notes say which hits are metadata-only, which
228
+ * titles repeat, which licences are not open.
229
+ */
230
+ export declare function findDatasets(client: CkanClient, params?: FindDatasetsParams): Promise<FindDatasetsAnswer>;
231
+ /** One portal's count. */
232
+ export interface PortalCount {
233
+ id: string;
234
+ title: string;
235
+ url: string;
236
+ /** Datasets matching now; absent when the portal failed. */
237
+ count?: number;
238
+ /** Why the portal could not be counted (HTTP 500, timeout, …). */
239
+ problem?: string;
240
+ /** The national catalogue, which harvests most of the others: never add it to a sum. */
241
+ national?: true;
242
+ }
243
+ export interface ComparePortalsAnswer extends Omit<AnswerBase, "ambiguous" | "portal"> {
244
+ /** Portals with at least one match. */
245
+ withMatches: number;
246
+ portals: PortalCount[];
247
+ }
248
+ /**
249
+ * How many datasets match a query (and a format) on every portal of the built-in list, or on the
250
+ * ones named, most first: one count request per portal, nothing else. A portal that fails is
251
+ * listed with its problem, not left out. GovData, the national catalogue, harvests most Länder
252
+ * portals: it is marked national and must never be added to the others.
253
+ */
254
+ export declare function comparePortals(client: CkanClient, params?: {
255
+ query?: string | undefined;
256
+ format?: string | undefined;
257
+ portals?: readonly string[] | undefined;
258
+ }): Promise<ComparePortalsAnswer>;
259
+ /** The distributions countDatasets counts, by name, with CKAN's facet field. */
260
+ export declare const FACETS: Readonly<{
261
+ organization: "organization";
262
+ group: "groups";
263
+ format: "res_format";
264
+ licence: "license_id";
265
+ tag: "tags";
266
+ type: "type";
267
+ documentType: "extras_registerobject_type";
268
+ }>;
269
+ export type FacetName = keyof typeof FACETS;
270
+ export declare const DEFAULT_FACET_VALUES = 10;
271
+ export declare const MAX_FACET_VALUES = 100;
272
+ /** The most years one count spreads over (one request each). */
273
+ export declare const MAX_YEARS = 20;
274
+ export interface CountDatasetsParams extends DatasetFilters {
275
+ facets?: readonly FacetName[] | undefined;
276
+ /** Values per facet. */
277
+ limit?: number | undefined;
278
+ /** Count per year from this year … */
279
+ yearFrom?: number | undefined;
280
+ /** … to this one (on dateField: created, or modified). */
281
+ yearTo?: number | undefined;
282
+ }
283
+ export interface FacetValue {
284
+ /** The value to filter by (a slug, a format, a licence's short name, a tag, a type). */
285
+ value: string;
286
+ /** The readable name, where it differs (an organization's or a group's title, a document type's name). */
287
+ label?: string;
288
+ /** Datasets, not files: a dataset with ten CSV files counts once for CSV. */
289
+ count: number;
290
+ /** The raw values folded into this one. */
291
+ variants?: string[];
292
+ /** The count is a sum over several spellings: a dataset with two of them counts twice. */
293
+ upperBound?: boolean;
294
+ }
295
+ export interface FacetCounts {
296
+ facet: FacetName;
297
+ values: FacetValue[];
298
+ /** False when there are more values than listed. */
299
+ complete: boolean;
300
+ /** Format values that name no clear format (MIME types, other vocabularies, combined values). */
301
+ other?: {
302
+ count: number;
303
+ variants: string[];
304
+ };
305
+ }
306
+ export interface CountDatasetsAnswer extends AnswerBase {
307
+ total: number;
308
+ facets: FacetCounts[];
309
+ /** Per year, on the date field, when asked. */
310
+ byYear?: {
311
+ year: number;
312
+ count: number;
313
+ }[];
314
+ }
315
+ /**
316
+ * How many datasets match on one portal, and how they spread over the facets asked for (no
317
+ * datasets fetched), and per year when asked (one count per year on the date field). The values
318
+ * are sorted by count (CKAN sends them unsorted); a format's spellings (CSV, csv, the EU URI) and a
319
+ * licence's (dl-de-by-2.0, dl-by-de/2.0, the URI) are folded into one value, their sum an upper
320
+ * bound; every value of those two is read, so nothing is cut before folding. For the others,
321
+ * complete false says there are more values than listed.
322
+ */
323
+ export declare function countDatasets(client: CkanClient, params?: CountDatasetsParams): Promise<CountDatasetsAnswer>;
324
+ export declare const DEFAULT_DOCUMENTS = 10;
325
+ export declare const MAX_DOCUMENTS = 50;
326
+ export interface FindHamburgDocumentsParams {
327
+ /** A document type: its index value (gutacht), its German name (Gutachten) or Hamburg's own value. */
328
+ documentType?: string | undefined;
329
+ query?: string | undefined;
330
+ /** An organization slug (allris, workflows, …): many records have none, so it drops them. */
331
+ organization?: string | undefined;
332
+ /** Entered the portal from this day on (YYYY-MM-DD; metadata_created). */
333
+ from?: string | undefined;
334
+ to?: string | undefined;
335
+ /** created (newest first, default), modified, relevance, title. */
336
+ sort?: DatasetSort | undefined;
337
+ offset?: number | undefined;
338
+ limit?: number | undefined;
339
+ }
340
+ /** One document of the Transparenzportal. */
341
+ export interface HamburgDocument {
342
+ name: string;
343
+ title?: string;
344
+ /** Its document types, by their German names. */
345
+ documentTypes: string[];
346
+ /** The publication date the record names (publishing_date). */
347
+ published?: string;
348
+ /** When it entered the portal (metadata_created). */
349
+ created?: string;
350
+ /** When it leaves the portal (offline_date). */
351
+ offline?: string;
352
+ organization?: string;
353
+ /** The office behind it (author). */
354
+ author?: string;
355
+ description?: string;
356
+ /** The authority's file number. */
357
+ fileReference?: string;
358
+ licence?: Licence;
359
+ attribution?: string;
360
+ /** Files: how many, how many PDFs, their known size. */
361
+ counts: {
362
+ files: number;
363
+ pdfs: number;
364
+ knownBytes: number;
365
+ };
366
+ /** The first files, with their links. */
367
+ files: FileEntry[];
368
+ }
369
+ export interface FindHamburgDocumentsAnswer extends AnswerBase {
370
+ total: number;
371
+ offset: number;
372
+ documents: HamburgDocument[];
373
+ nextOffset?: number;
374
+ }
375
+ /**
376
+ * Documents of the Transparenzportal Hamburg (the register the Hamburg Transparency Act requires):
377
+ * by document type (sent as the stemmed value its search index holds, the only one that works),
378
+ * topic and the day they entered the portal, newest first, a page at a time. Each with its
379
+ * publication date, types, office, licence and attribution text, and its files. Always on Hamburg's
380
+ * portal, whatever portal the client talks to.
381
+ */
382
+ export declare function findHamburgDocuments(client: CkanClient, params?: FindHamburgDocumentsParams): Promise<FindHamburgDocumentsAnswer>;
383
+ /** A dataset's slug or id from itself or a link to its page (…/dataset/NAME). */
384
+ export declare function datasetIdFrom(value: string): string;
385
+ export interface DatasetRecord {
386
+ name: string;
387
+ id?: string;
388
+ title?: string;
389
+ type?: string;
390
+ description?: string;
391
+ organization?: {
392
+ name: string;
393
+ title?: string;
394
+ };
395
+ author?: string;
396
+ maintainer?: string;
397
+ groups: {
398
+ name: string;
399
+ title?: string;
400
+ }[];
401
+ tags: string[];
402
+ modified?: string;
403
+ created?: string;
404
+ /** The publisher's page for it. */
405
+ landingPage?: string;
406
+ /** Its licence, and any other its files name. */
407
+ licences: Licence[];
408
+ attribution?: string;
409
+ /** Selected extras: publication date, document types, temporal coverage, file number, harvest source, publisher. */
410
+ details: Record<string, JsonValue>;
411
+ files: FileEntry[];
412
+ }
413
+ export interface GetDatasetAnswer extends Omit<AnswerBase, "ambiguous"> {
414
+ dataset?: DatasetRecord;
415
+ }
416
+ /**
417
+ * One record by its name (slug), id or the link to its page, on one portal: organization, author,
418
+ * groups, tags, dates, licences and attribution, selected details, and every file with its kind,
419
+ * format, size and licence. An unknown name is an empty answer that says where to take one from.
420
+ */
421
+ export declare function getDataset(client: CkanClient, params: {
422
+ id: string;
423
+ portal?: string | undefined;
424
+ }): Promise<GetDatasetAnswer>;
425
+ export interface ListOrganizationsAnswer extends Omit<AnswerBase, "ambiguous"> {
426
+ kind: "organization" | "group";
427
+ /** Organizations (or groups) with datasets on the portal (all, not only those listed). */
428
+ total: number;
429
+ organizations: OrganizationEntry[];
430
+ }
431
+ /**
432
+ * A portal's organizations (publishers), or its groups (themes), most datasets first, with their
433
+ * slugs for the filters; with a name, those whose slug or title contains it. Counted from the
434
+ * search's facet, so only those with datasets.
435
+ */
436
+ export declare function listOrganizations(client: CkanClient, params?: {
437
+ portal?: string | undefined;
438
+ kind?: "organization" | "group" | undefined;
439
+ name?: string | undefined;
440
+ }): Promise<ListOrganizationsAnswer>;
441
+ export {};