@maschinenlesbar.org/ckan-cli 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.d.ts +25 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/client.js +63 -22
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.d.ts +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/index.js +2 -1
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.d.ts +441 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/operations.js +1211 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.d.ts +2 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-client/dist/src/portals.js +2 -2
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.d.ts +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/index.js +1 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.d.ts +80 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/dist/src/log.js +124 -0
- package/dist/node_modules/@maschinenlesbar.org/ckan-log/package.json +25 -0
- package/dist/src/cli/index.js +3 -1
- package/dist/src/cli/io.d.ts +17 -0
- package/dist/src/cli/io.js +14 -0
- package/dist/src/cli/limits.d.ts +19 -0
- package/dist/src/cli/limits.js +53 -0
- package/dist/src/cli/program.js +3 -0
- package/dist/src/cli/run.js +9 -1
- package/dist/src/cli/shared.d.ts +4 -0
- package/dist/src/cli/shared.js +16 -1
- package/node_modules/commander/Readme.md +6 -32
- package/node_modules/commander/index.js +1 -1
- package/node_modules/commander/lib/command.js +22 -9
- package/node_modules/commander/lib/help.js +6 -2
- package/node_modules/commander/lib/option.js +1 -1
- package/node_modules/commander/package.json +2 -1
- package/node_modules/commander/typings/index.d.ts +6 -3
- package/package.json +2 -2
|
@@ -0,0 +1,1211 @@
|
|
|
1
|
+
// The use cases of the skills as library functions, built on the client: each composes its calls
|
|
2
|
+
// and carries the traps the skills document, so no caller (the CLI, the MCP server's tools, a
|
|
3
|
+
// library user) has to know them:
|
|
4
|
+
// - findDatasets (ckan-dataset-finder): datasets for a topic and friendly filters (organization
|
|
5
|
+
// and group by name, tag, file format in every spelling, record type, a date range, only
|
|
6
|
+
// datasets with files) on one portal, a page at a time, each with its formats folded and its
|
|
7
|
+
// licences;
|
|
8
|
+
// - comparePortals (ckan-dataset-finder, ckan-catalogue-stats): how many datasets match on every
|
|
9
|
+
// portal of the built-in list at once, GovData (which harvests the others) apart;
|
|
10
|
+
// - countDatasets (ckan-catalogue-stats): how many datasets match on one portal, how they spread
|
|
11
|
+
// over organizations, groups, formats, licences, tags, record and document types (sorted, the
|
|
12
|
+
// spellings of one format or licence folded), and per year;
|
|
13
|
+
// - findHamburgDocuments (ckan-hamburg-transparency): the Transparenzportal Hamburg's documents by
|
|
14
|
+
// document type (the stemmed value that works), topic and date, with their publication date,
|
|
15
|
+
// licence, attribution text and files;
|
|
16
|
+
// - getDataset and listOrganizations, for what those don't reach: one record in full, and a
|
|
17
|
+
// portal's organizations or groups with their dataset counts.
|
|
18
|
+
//
|
|
19
|
+
// A portal is the client's own, or one of the built-in list by its id (never a URL: the list is
|
|
20
|
+
// what a caller may reach). The rules they keep: no bare "not found" (an empty answer says what was
|
|
21
|
+
// searched and what to try); several organizations or groups for one name are reported as
|
|
22
|
+
// ambiguous, never guessed; each answer is a named type (public API of the CLI package: a changed
|
|
23
|
+
// shape is a `feat(client)!:` commit); every request an answer made is kept (`requestsOf(answer)`),
|
|
24
|
+
// as the client sends it, for its source.
|
|
25
|
+
import { actionRequest, packageSearchQuery } from "./client.js";
|
|
26
|
+
import { CkanActionError, CkanApiError, CkanError, CkanValidationError } from "./errors.js";
|
|
27
|
+
import { mapLimit, portalKey, problemOf } from "./portals.js";
|
|
28
|
+
import { PORTALS } from "./portals-list.js";
|
|
29
|
+
// ---- portals ---------------------------------------------------------------------------------
|
|
30
|
+
/** The Action API's path under a portal's site root. */
|
|
31
|
+
const ACTION_PATH = "/api/3/action/";
|
|
32
|
+
/** A portal of the built-in list by its id (any case); a URL or an unknown id is refused. */
|
|
33
|
+
export function portalById(id) {
|
|
34
|
+
const wanted = typeof id === "string" ? id.trim().toLowerCase() : "";
|
|
35
|
+
const portal = PORTALS.find((p) => p.id === wanted);
|
|
36
|
+
if (portal === undefined) {
|
|
37
|
+
throw new CkanValidationError(`Invalid portal: ${JSON.stringify(String(id).slice(0, 80))} is not an id of the built-in list (${PORTALS.map((p) => p.id).join(", ")}).`);
|
|
38
|
+
}
|
|
39
|
+
return portal;
|
|
40
|
+
}
|
|
41
|
+
/** The portal of the list a client talks to, when it is one. */
|
|
42
|
+
function portalOfClient(client) {
|
|
43
|
+
return PORTALS.find((p) => portalKey(p.url) === client.siteKey);
|
|
44
|
+
}
|
|
45
|
+
/** The site for `portalId` (an id of the list), or the client's own when none is given. */
|
|
46
|
+
function siteOf(client, portalId) {
|
|
47
|
+
if (portalId === undefined) {
|
|
48
|
+
const own = portalOfClient(client);
|
|
49
|
+
return own === undefined ? { client, prefix: "" } : { client, portal: own, prefix: "" };
|
|
50
|
+
}
|
|
51
|
+
const portal = portalById(portalId);
|
|
52
|
+
if (portalKey(portal.url) === client.siteKey)
|
|
53
|
+
return { client, portal, prefix: "" };
|
|
54
|
+
return { client: client.withBaseUrl(portal.url), portal, prefix: `${portal.url.replace(/\/+$/, "")}${ACTION_PATH}` };
|
|
55
|
+
}
|
|
56
|
+
const portalRef = (site) => (site.portal === undefined ? {} : { id: site.portal.id, title: site.portal.title, url: site.portal.url });
|
|
57
|
+
// ---- the requests an answer made -------------------------------------------------------------
|
|
58
|
+
const REQUESTS = new WeakMap();
|
|
59
|
+
/**
|
|
60
|
+
* The requests the operation that returned `answer` made, in order, each as the client sends it:
|
|
61
|
+
* under the client's own Action API (`package_search?q=…`), or as an absolute URL on another
|
|
62
|
+
* portal of the list. What its `source` names. Empty for an object no operation returned.
|
|
63
|
+
*/
|
|
64
|
+
export function requestsOf(answer) {
|
|
65
|
+
return REQUESTS.get(answer) ?? [];
|
|
66
|
+
}
|
|
67
|
+
/** The requests of one operation call. */
|
|
68
|
+
class Trace {
|
|
69
|
+
requests = [];
|
|
70
|
+
async search(site, params) {
|
|
71
|
+
this.requests.push(`${site.prefix}${actionRequest("package_search", packageSearchQuery(params))}`);
|
|
72
|
+
return site.client.packageSearch(params);
|
|
73
|
+
}
|
|
74
|
+
done(answer) {
|
|
75
|
+
REQUESTS.set(answer, [...this.requests]);
|
|
76
|
+
return answer;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
// ---- reading CKAN's JSON ---------------------------------------------------------------------
|
|
80
|
+
const isObject = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
81
|
+
const textOf = (value) => (typeof value === "string" && value.trim() !== "" ? value.trim() : undefined);
|
|
82
|
+
const objectsOf = (value) => (Array.isArray(value) ? value.filter(isObject) : []);
|
|
83
|
+
/** A dataset's `extras` as a map. */
|
|
84
|
+
function extrasOf(dataset) {
|
|
85
|
+
const extras = new Map();
|
|
86
|
+
for (const extra of objectsOf(dataset["extras"])) {
|
|
87
|
+
const key = textOf(extra["key"]);
|
|
88
|
+
const value = textOf(extra["value"]);
|
|
89
|
+
if (key !== undefined && value !== undefined)
|
|
90
|
+
extras.set(key, value);
|
|
91
|
+
}
|
|
92
|
+
return extras;
|
|
93
|
+
}
|
|
94
|
+
/** Whitespace collapsed, cut at `max` characters with an ellipsis. */
|
|
95
|
+
function excerpt(text, max) {
|
|
96
|
+
if (text === undefined)
|
|
97
|
+
return undefined;
|
|
98
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
99
|
+
if (flat === "")
|
|
100
|
+
return undefined;
|
|
101
|
+
return flat.length > max ? `${flat.slice(0, max - 1).trimEnd()}…` : flat;
|
|
102
|
+
}
|
|
103
|
+
/** HTML or Markdown-ish text (a description can arrive as `<p>…</p>`) as plain text, cut at `max`. */
|
|
104
|
+
function plainText(html, max = 1000) {
|
|
105
|
+
if (html === undefined)
|
|
106
|
+
return undefined;
|
|
107
|
+
const text = html
|
|
108
|
+
.replace(/<[^>]*>/g, " ")
|
|
109
|
+
.replace(/ /g, " ")
|
|
110
|
+
.replace(/</g, "<")
|
|
111
|
+
.replace(/>/g, ">")
|
|
112
|
+
.replace(/"/g, '"')
|
|
113
|
+
.replace(/'/g, "'")
|
|
114
|
+
.replace(/&/g, "&");
|
|
115
|
+
return excerpt(text, max);
|
|
116
|
+
}
|
|
117
|
+
/** Folded for comparing names: lower case, no accents (ü as u, like CKAN's slugs), ß as ss. */
|
|
118
|
+
export function foldName(text) {
|
|
119
|
+
return text.normalize("NFKD").replace(/\p{M}/gu, "").replace(/ß/g, "ss").toLowerCase().replace(/\s+/g, " ").trim();
|
|
120
|
+
}
|
|
121
|
+
/** A Solr phrase: the value in double quotes, its quotes and backslashes escaped. */
|
|
122
|
+
const phrase = (value) => `"${value.replace(/[\\"]/g, "\\$&")}"`;
|
|
123
|
+
// ---- formats ---------------------------------------------------------------------------------
|
|
124
|
+
/** The EU file-type vocabulary, the spelling of most format values on GovData and daten.bw. */
|
|
125
|
+
export const EU_FILE_TYPE = "http://publications.europa.eu/resource/authority/file-type/";
|
|
126
|
+
/** Free-text and MIME spellings of a format, by what they mean. */
|
|
127
|
+
const FORMAT_ALIASES = {
|
|
128
|
+
SHAPE: "SHP",
|
|
129
|
+
SHAPEFILE: "SHP",
|
|
130
|
+
SHAPEFILES: "SHP",
|
|
131
|
+
"GEZIPPTE SHAPE": "SHP",
|
|
132
|
+
"ESRI SHAPEFILE": "SHP",
|
|
133
|
+
"GEZIPPTE SHAPEDATEI": "SHP",
|
|
134
|
+
"GEZIPPTE SHAPEDATEIEN": "SHP",
|
|
135
|
+
"GML+XML": "GML",
|
|
136
|
+
GEOPACKAGE: "GPKG",
|
|
137
|
+
"APPLICATION/GEOPACKAGE+SQLITE3": "GPKG",
|
|
138
|
+
JPG: "JPEG",
|
|
139
|
+
"JSON-LD": "JSON_LD",
|
|
140
|
+
"TEXT/CSV": "CSV",
|
|
141
|
+
"APPLICATION/JSON": "JSON",
|
|
142
|
+
"APPLICATION/GEO+JSON": "GEOJSON",
|
|
143
|
+
"APPLICATION/VND.GEO+JSON": "GEOJSON",
|
|
144
|
+
"APPLICATION/ZIP": "ZIP",
|
|
145
|
+
"APPLICATION/PDF": "PDF",
|
|
146
|
+
"APPLICATION/XML": "XML",
|
|
147
|
+
"TEXT/XML": "XML",
|
|
148
|
+
"TEXT/HTML": "HTML",
|
|
149
|
+
"APPLICATION/VND.MS-EXCEL": "XLS",
|
|
150
|
+
"APPLICATION/VND.OPENXMLFORMATS-OFFICEDOCUMENT.SPREADSHEETML.SHEET": "XLSX",
|
|
151
|
+
};
|
|
152
|
+
/** Values in the format field that name no format. */
|
|
153
|
+
const NOT_FORMATS = new Set(["DOWNLOAD", "LINK", "WEBSITE", "WEBSEITE", "WEB", "DATEI", "FILE", "DATA", "DATEN"]);
|
|
154
|
+
/** Formats that are services, not files. */
|
|
155
|
+
export const SERVICE_FORMATS = new Set(["WMS", "WFS", "WCS", "WMTS", "OAF", "ATOM", "CSW", "SOS", "WPS", "MQTT", "STA"]);
|
|
156
|
+
/**
|
|
157
|
+
* A format value as one short name: the EU URI's tail (`…/file-type/CSV` is CSV), upper case (csv
|
|
158
|
+
* and CSV are one), the EU `_SRVC` suffix dropped (WMS_SRVC is WMS), free-text and MIME spellings
|
|
159
|
+
* mapped (Shapefiles is SHP, text/csv is CSV). Undefined for a value that names no clear format:
|
|
160
|
+
* another vocabulary's URI, an unknown MIME type, a combined value ("Shape, CSV"), a word like
|
|
161
|
+
* "Download".
|
|
162
|
+
*/
|
|
163
|
+
export function formatOf(raw) {
|
|
164
|
+
const text = textOf(raw);
|
|
165
|
+
if (text === undefined)
|
|
166
|
+
return undefined;
|
|
167
|
+
let upper = text.toUpperCase();
|
|
168
|
+
const alias = FORMAT_ALIASES[upper];
|
|
169
|
+
if (alias !== undefined)
|
|
170
|
+
return alias;
|
|
171
|
+
if (/^HTTPS?:\/\//.test(upper)) {
|
|
172
|
+
const media = /\/ASSIGNMENTS\/MEDIA-TYPES\/(.+)$/.exec(upper);
|
|
173
|
+
if (media !== null)
|
|
174
|
+
return FORMAT_ALIASES[media[1]];
|
|
175
|
+
// ArcGIS Hub's DCAT feeds name the format at the end of their own URI (…/ftype/CSV).
|
|
176
|
+
const arcgis = /\/FTYPE\/([A-Z0-9_]+)$/.exec(upper);
|
|
177
|
+
if (arcgis !== null)
|
|
178
|
+
return formatOf(arcgis[1]);
|
|
179
|
+
if (!upper.startsWith(EU_FILE_TYPE.toUpperCase()))
|
|
180
|
+
return undefined;
|
|
181
|
+
upper = upper.slice(EU_FILE_TYPE.length);
|
|
182
|
+
}
|
|
183
|
+
upper = upper.replace(/_SRVC$/, "");
|
|
184
|
+
const mapped = FORMAT_ALIASES[upper] ?? upper;
|
|
185
|
+
if (!/^[A-Z0-9_]{1,12}$/.test(mapped) || NOT_FORMATS.has(mapped))
|
|
186
|
+
return undefined;
|
|
187
|
+
return mapped;
|
|
188
|
+
}
|
|
189
|
+
/** A requested format as `formatOf` names it: letters, digits, `_`, `-`, `+`, `/` or `.`, at most 40 characters. */
|
|
190
|
+
function wantedFormat(value) {
|
|
191
|
+
const text = value.trim();
|
|
192
|
+
const format = /^[A-Za-z0-9_+\-/.]{1,40}$/.test(text) ? formatOf(text) : undefined;
|
|
193
|
+
if (format === undefined) {
|
|
194
|
+
throw new CkanValidationError(`Invalid format: expected a format name such as CSV, GEOJSON or SHP, got ${JSON.stringify(text.slice(0, 40))}.`);
|
|
195
|
+
}
|
|
196
|
+
return format;
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* The spellings portals store a format under: upper case (NRW, München), lower case (Hamburg; Berlin
|
|
200
|
+
* has both as separate values, and its filter is case-sensitive), the EU URI (GovData, daten.bw),
|
|
201
|
+
* free-text and MIME aliases.
|
|
202
|
+
*/
|
|
203
|
+
export function formatSpellings(format) {
|
|
204
|
+
const spellings = new Set([format, format.toLowerCase(), `${EU_FILE_TYPE}${format}`]);
|
|
205
|
+
if (SERVICE_FORMATS.has(format))
|
|
206
|
+
spellings.add(`${EU_FILE_TYPE}${format}_SRVC`);
|
|
207
|
+
for (const [alias, meaning] of Object.entries(FORMAT_ALIASES)) {
|
|
208
|
+
if (meaning !== format)
|
|
209
|
+
continue;
|
|
210
|
+
spellings.add(alias.includes("/") ? alias.toLowerCase() : alias.charAt(0) + alias.slice(1).toLowerCase());
|
|
211
|
+
if (!alias.includes("/")) {
|
|
212
|
+
// Open.NRW writes them in capitals (GEZIPPTE SHAPEDATEI), Hamburg in lower case (shapefiles).
|
|
213
|
+
spellings.add(alias);
|
|
214
|
+
spellings.add(alias.toLowerCase());
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
if (format === "GEOJSON")
|
|
218
|
+
spellings.add("GeoJSON");
|
|
219
|
+
if (format === "GPKG")
|
|
220
|
+
spellings.add("GeoPackage");
|
|
221
|
+
return [...spellings];
|
|
222
|
+
}
|
|
223
|
+
/** The Solr filter for datasets with a file in `format`, in any of its spellings. */
|
|
224
|
+
export function formatFilter(format) {
|
|
225
|
+
return `res_format:(${formatSpellings(format).map(phrase).join(" OR ")})`;
|
|
226
|
+
}
|
|
227
|
+
/** Licences by their DCAT-AP.de id (http://dcat-ap.de/def/licenses/…), `2_0` read as `2.0`. */
|
|
228
|
+
const LICENCES = {
|
|
229
|
+
"dl-by-de/2.0": ["DL-DE BY 2.0", true],
|
|
230
|
+
"dl-zero-de/2.0": ["DL-DE Zero 2.0", true],
|
|
231
|
+
"dl-by-de/1.0": ["DL-DE BY 1.0", true],
|
|
232
|
+
"dl-by-nc-de/1.0": ["DL-DE BY-NC 1.0", false],
|
|
233
|
+
"cc-by/4.0": ["CC BY 4.0", true],
|
|
234
|
+
"cc-by": ["CC BY", true],
|
|
235
|
+
"cc-by-de/3.0": ["CC BY 3.0 DE", true],
|
|
236
|
+
"cc-by-sa/4.0": ["CC BY-SA 4.0", true],
|
|
237
|
+
"cc-by-sa": ["CC BY-SA", true],
|
|
238
|
+
"cc-by-sa-de/3.0": ["CC BY-SA 3.0 DE", true],
|
|
239
|
+
"cc-by-nd/4.0": ["CC BY-ND 4.0", false],
|
|
240
|
+
"cc-by-nc/4.0": ["CC BY-NC 4.0", false],
|
|
241
|
+
"cc-by-nc": ["CC BY-NC", false],
|
|
242
|
+
"cc-zero": ["CC0 1.0", true],
|
|
243
|
+
odbl: ["ODbL", true],
|
|
244
|
+
"geonutz/20130319": ["GeoNutzV (Geodaten des Bundes)", true],
|
|
245
|
+
"geonutz/20131001": ["GeoNutzV (Geodaten des Bundes)", true],
|
|
246
|
+
"official-work": ["Amtliches Werk (§ 5 UrhG)", true],
|
|
247
|
+
"other-closed": ["other, closed (not an open licence)", false],
|
|
248
|
+
"other-open": ["other open licence", true],
|
|
249
|
+
"other-opensource": ["other open-source licence", true],
|
|
250
|
+
"other-freeware": ["freeware", false],
|
|
251
|
+
"other-commercial": ["commercial", false],
|
|
252
|
+
};
|
|
253
|
+
/** A licence id in the DCAT-AP.de form, whatever spelling a portal uses (dl-de-by-2.0, cc-by-4.0, geoNutz-20130319). */
|
|
254
|
+
function licenceKey(raw) {
|
|
255
|
+
let key = raw.toLowerCase().replace(/\/+$/, "").replace(/^https?:\/\/dcat-ap\.de\/def\/licenses\//, "").replace(/_/g, ".");
|
|
256
|
+
key = key.replace(/^dl-de-(by|zero|by-nc)-(\d\.\d)$/, "dl-$1-de/$2");
|
|
257
|
+
key = key.replace(/^dl-(by|zero|by-nc)-de-(\d\.\d)$/, "dl-$1-de/$2");
|
|
258
|
+
key = key.replace(/^(cc-by(?:-sa|-nd|-nc)?)-(\d\.\d)$/, "$1/$2");
|
|
259
|
+
key = key.replace(/^geonutz-(\d{8})$/, "geonutz/$1");
|
|
260
|
+
key = key.replace(/^cc-zero\/1\.0$/, "cc-zero");
|
|
261
|
+
return key;
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* A licence with a short name; undefined for an empty value. The spellings of one licence across
|
|
265
|
+
* portals (dl-de-by-2.0, dl-by-de/2.0, the DCAT-AP.de URI, 2_0) get one label; `title` (the
|
|
266
|
+
* portal's license_title) names an unknown one.
|
|
267
|
+
*/
|
|
268
|
+
export function licenceOf(raw, title) {
|
|
269
|
+
const id = textOf(raw);
|
|
270
|
+
if (id === undefined)
|
|
271
|
+
return undefined;
|
|
272
|
+
const known = LICENCES[licenceKey(id)];
|
|
273
|
+
if (known !== undefined)
|
|
274
|
+
return { id, label: known[0], open: known[1] };
|
|
275
|
+
const lower = id.toLowerCase().replace(/\/+$/, "");
|
|
276
|
+
const govdata = /govdata\.de\/dl-de\/(by|zero|by-nc)-(\d)-(\d)$/.exec(lower);
|
|
277
|
+
if (govdata !== null) {
|
|
278
|
+
const label = `DL-DE ${govdata[1] === "zero" ? "Zero" : govdata[1].toUpperCase()} ${govdata[2]}.${govdata[3]}`;
|
|
279
|
+
return { id, label, open: govdata[1] !== "by-nc" };
|
|
280
|
+
}
|
|
281
|
+
const cc = /creativecommons\.org\/licenses\/([a-z-]+)\/(\d\.\d)/.exec(lower);
|
|
282
|
+
if (cc !== null) {
|
|
283
|
+
const kind = cc[1];
|
|
284
|
+
return { id, label: `CC ${kind.toUpperCase()} ${cc[2]}`, open: !/nc|nd/.test(kind) };
|
|
285
|
+
}
|
|
286
|
+
if (/creativecommons\.org\/publicdomain\/zero\//.test(lower))
|
|
287
|
+
return { id, label: "CC0 1.0", open: true };
|
|
288
|
+
const named = textOf(title);
|
|
289
|
+
if (named !== undefined && named !== id)
|
|
290
|
+
return { id, label: excerpt(named, 120) };
|
|
291
|
+
const tail = id.replace(/\/+$/, "").split(/[/#]/).pop() ?? id;
|
|
292
|
+
return { id, label: tail === "" ? id : tail };
|
|
293
|
+
}
|
|
294
|
+
/** The distinct licences (by label) of `pairs` of id and title, in the order first seen. */
|
|
295
|
+
function distinctLicences(pairs) {
|
|
296
|
+
const seen = new Map();
|
|
297
|
+
for (const [raw, title] of pairs) {
|
|
298
|
+
const licence = licenceOf(raw, title);
|
|
299
|
+
if (licence !== undefined && !seen.has(licence.label))
|
|
300
|
+
seen.set(licence.label, licence);
|
|
301
|
+
}
|
|
302
|
+
return [...seen.values()];
|
|
303
|
+
}
|
|
304
|
+
/** A dataset's licences: its own (license_id), then any other its files name. */
|
|
305
|
+
function licencesOf(dataset) {
|
|
306
|
+
return distinctLicences([[dataset["license_id"], dataset["license_title"]], ...objectsOf(dataset["resources"]).map((r) => [r["license"], undefined])]);
|
|
307
|
+
}
|
|
308
|
+
/** The attribution text a record asks for: Hamburg's terms_of_use (a JSON string), else a DCAT extra. */
|
|
309
|
+
function attributionOf(dataset, extras) {
|
|
310
|
+
const terms = extras.get("terms_of_use");
|
|
311
|
+
if (terms !== undefined) {
|
|
312
|
+
try {
|
|
313
|
+
const parsed = JSON.parse(terms);
|
|
314
|
+
if (isObject(parsed)) {
|
|
315
|
+
const text = plainText(textOf(parsed["attribution_text"]), 500);
|
|
316
|
+
if (text !== undefined)
|
|
317
|
+
return text;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
catch {
|
|
321
|
+
// Not JSON: no attribution from it.
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return plainText(extras.get("licenseAttributionByText") ?? textOf(dataset["licenseAttributionByText"]), 500);
|
|
325
|
+
}
|
|
326
|
+
/** The link with a percent-encoded query (`%3f`, as harvested) read as a query. */
|
|
327
|
+
function decodedLink(url) {
|
|
328
|
+
try {
|
|
329
|
+
return decodeURI(url.replace(/%3f/gi, "?").replace(/%3d/gi, "=").replace(/%26/gi, "&"));
|
|
330
|
+
}
|
|
331
|
+
catch {
|
|
332
|
+
return url;
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
/** The extension at the end of a link's path (`.csv.gz` is CSV), upper case, or undefined. */
|
|
336
|
+
export function extensionOf(url) {
|
|
337
|
+
const path = decodedLink(url).split(/[?#]/)[0] ?? "";
|
|
338
|
+
const match = /\.([a-z0-9]{2,8})(?:\.(?:gz|zip|bz2))?$/i.exec(path.split("/").pop() ?? "");
|
|
339
|
+
if (match === null)
|
|
340
|
+
return undefined;
|
|
341
|
+
const ext = match[1].toUpperCase();
|
|
342
|
+
return FORMAT_ALIASES[ext] ?? ext;
|
|
343
|
+
}
|
|
344
|
+
/** Whether the link carries a key or token in its query (as published). */
|
|
345
|
+
export function hasKeyInLink(url) {
|
|
346
|
+
return /[?&](api[-_]?key|apikey|token|access_token|key)=/i.test(decodedLink(url));
|
|
347
|
+
}
|
|
348
|
+
/** The kind of a resource's link (see LinkKind), from the link and its format. */
|
|
349
|
+
export function linkKind(url, format) {
|
|
350
|
+
const link = decodedLink(url);
|
|
351
|
+
const [path = "", query = ""] = link.split(/\?/, 2);
|
|
352
|
+
if (/\{[^}]*\}|%7b/i.test(url) || /(^|[_\-/.])(yyyy|yyyymm|yyyymmdd|jjjj|jjjjmm)([_\-/.]|$)/i.test(path))
|
|
353
|
+
return "template";
|
|
354
|
+
// Cut off by a harvester: a resource link without its id, or with part of it.
|
|
355
|
+
if (/\/resource\/[0-9a-f-]{1,35}$/i.test(path) || /\/dataset\/[^/]+\/(?:res|reso|resou|resour|resourc|resource)(?:-\d+)?$/i.test(path))
|
|
356
|
+
return "broken";
|
|
357
|
+
if (/[?&]request=getfeature/i.test(`?${query}`))
|
|
358
|
+
return "generated";
|
|
359
|
+
if (/(^|&)service=(wms|wfs|wcs|wmts|csw|sos|wps)(&|$)/i.test(query) || /(^|&)request=/i.test(query))
|
|
360
|
+
return "service";
|
|
361
|
+
if (format !== undefined && SERVICE_FORMATS.has(format))
|
|
362
|
+
return "service";
|
|
363
|
+
if (/\/dataset\/[^/]+\/resource\/[0-9a-f-]{36}$/i.test(path))
|
|
364
|
+
return "page";
|
|
365
|
+
const ext = extensionOf(path);
|
|
366
|
+
if (ext !== undefined)
|
|
367
|
+
return /^(HTML?|PHP|ASPX?|JSP)$/.test(ext) ? "page" : "file";
|
|
368
|
+
if (hasKeyInLink(url))
|
|
369
|
+
return "api";
|
|
370
|
+
if (/\/(download|export|exports|dump)(\/|$)/i.test(path))
|
|
371
|
+
return "generated";
|
|
372
|
+
if (path.endsWith("/") || format === "HTML")
|
|
373
|
+
return "page";
|
|
374
|
+
return query === "" ? "page" : "api";
|
|
375
|
+
}
|
|
376
|
+
/** The kinds of link a download can fetch. */
|
|
377
|
+
const FETCHABLE = new Set(["file", "generated"]);
|
|
378
|
+
/** A resource as a file entry; undefined without a URL. */
|
|
379
|
+
function fileOf(resource) {
|
|
380
|
+
const url = textOf(resource["url"]) ?? textOf(resource["download_url"]) ?? textOf(resource["access_url"]);
|
|
381
|
+
if (url === undefined)
|
|
382
|
+
return undefined;
|
|
383
|
+
const rawFormat = textOf(resource["format"]);
|
|
384
|
+
const format = formatOf(rawFormat);
|
|
385
|
+
const entry = { url, kind: linkKind(url, format) };
|
|
386
|
+
const id = textOf(resource["id"]);
|
|
387
|
+
if (id !== undefined)
|
|
388
|
+
entry.id = id;
|
|
389
|
+
const name = textOf(resource["name"]) ?? textOf(resource["description"]);
|
|
390
|
+
if (name !== undefined)
|
|
391
|
+
entry.name = excerpt(name, 200);
|
|
392
|
+
const shown = format ?? rawFormat;
|
|
393
|
+
if (shown !== undefined)
|
|
394
|
+
entry.format = excerpt(shown, 100);
|
|
395
|
+
const extension = entry.kind === "file" ? extensionOf(url) : undefined;
|
|
396
|
+
if (extension !== undefined && format !== undefined && extension !== format)
|
|
397
|
+
entry.extension = extension;
|
|
398
|
+
const size = resource["size"];
|
|
399
|
+
if (typeof size === "number" && Number.isSafeInteger(size) && size > 0)
|
|
400
|
+
entry.size = size;
|
|
401
|
+
const licence = licenceOf(resource["license"]);
|
|
402
|
+
if (licence !== undefined)
|
|
403
|
+
entry.licence = licence;
|
|
404
|
+
const modified = textOf(resource["last_modified"]) ?? textOf(resource["modified"]);
|
|
405
|
+
if (modified !== undefined)
|
|
406
|
+
entry.modified = modified;
|
|
407
|
+
if (hasKeyInLink(url))
|
|
408
|
+
entry.keyInLink = true;
|
|
409
|
+
return entry;
|
|
410
|
+
}
|
|
411
|
+
// ---- Hamburg's document types ----------------------------------------------------------------
|
|
412
|
+
/**
|
|
413
|
+
* The Transparenzportal Hamburg's document types (Informationsgegenstände), by the value its search
|
|
414
|
+
* index holds: Solr stems the stored words, so `extras_registerobject_type:gutacht` finds the
|
|
415
|
+
* Gutachten, while the published values (vertraege_oeff_interesse) and the stored ones
|
|
416
|
+
* (vertraegeoeffinteresse) find nothing, and some full words over-match (verwaltungsvorschriften
|
|
417
|
+
* finds the Dienstanweisungen too).
|
|
418
|
+
*/
|
|
419
|
+
export const DOCUMENT_TYPES = Object.freeze({
|
|
420
|
+
beschluss: "Beschlüsse",
|
|
421
|
+
bauleitplan: "Bauleitpläne",
|
|
422
|
+
baugenehm: "Baugenehmigungen",
|
|
423
|
+
statist: "Statistiken",
|
|
424
|
+
geodat: "Geodaten",
|
|
425
|
+
senatpetitum: "Senatspetita",
|
|
426
|
+
messung: "Messungen",
|
|
427
|
+
senatmitteil: "Senatsmitteilungen",
|
|
428
|
+
andereveroffentlichungspflicht: "andere Veröffentlichungspflicht",
|
|
429
|
+
ohneveroffentlichungspflicht: "ohne Veröffentlichungspflicht",
|
|
430
|
+
vertrageoffinteress: "Verträge von öffentlichem Interesse",
|
|
431
|
+
gutacht: "Gutachten",
|
|
432
|
+
vergleichbar: "vergleichbare Informationen",
|
|
433
|
+
verwaltungsvorschrift: "Verwaltungsvorschriften",
|
|
434
|
+
dienstanweis: "Dienstanweisungen",
|
|
435
|
+
daseinsvorsorg: "Daseinsvorsorge",
|
|
436
|
+
verwaltungsbelang: "Verwaltungsbelange",
|
|
437
|
+
stadtischebeteil: "städtische Beteiligungen",
|
|
438
|
+
baumkatast: "Baumkataster",
|
|
439
|
+
subvention: "Subventionen",
|
|
440
|
+
});
|
|
441
|
+
/** A word with the umlauts as Hamburg's values write them out (ae, oe, ue) taken back to a, o, u. */
|
|
442
|
+
const unUmlaut = (text) => foldName(text).replace(/ae/g, "a").replace(/oe/g, "o").replace(/ue/g, "u").replace(/[^a-z]/g, "");
|
|
443
|
+
/**
|
|
444
|
+
* A document type's index value from that value, its German name, Hamburg's stored value
|
|
445
|
+
* (vertraegeoeffinteresse) or its published one (vertraege_oeff_interesse); undefined if none fits.
|
|
446
|
+
*/
|
|
447
|
+
export function documentType(value) {
|
|
448
|
+
const folded = unUmlaut(value);
|
|
449
|
+
if (folded === "")
|
|
450
|
+
return undefined;
|
|
451
|
+
let best;
|
|
452
|
+
let bestLength = 0;
|
|
453
|
+
let tie = false;
|
|
454
|
+
for (const [stem, label] of Object.entries(DOCUMENT_TYPES)) {
|
|
455
|
+
if (folded === stem || folded === unUmlaut(label))
|
|
456
|
+
return stem;
|
|
457
|
+
// A stored or published value starts with the stem, give or take the last letters (senatpetita).
|
|
458
|
+
let common = 0;
|
|
459
|
+
while (common < stem.length && common < folded.length && stem[common] === folded[common])
|
|
460
|
+
common++;
|
|
461
|
+
if (common < Math.min(stem.length, 6) || (common !== stem.length && folded.length - common > 3))
|
|
462
|
+
continue;
|
|
463
|
+
if (common > bestLength) {
|
|
464
|
+
best = stem;
|
|
465
|
+
bestLength = common;
|
|
466
|
+
tie = false;
|
|
467
|
+
}
|
|
468
|
+
else if (common === bestLength) {
|
|
469
|
+
tie = true;
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
// Two types fit as well ("verwaltung"): none is guessed.
|
|
473
|
+
return tie ? undefined : best;
|
|
474
|
+
}
|
|
475
|
+
/** The German names of a record's stored type value ("dienstanweisungen, vergleichbar"). */
|
|
476
|
+
function documentTypesOf(stored) {
|
|
477
|
+
if (stored === undefined)
|
|
478
|
+
return [];
|
|
479
|
+
return stored
|
|
480
|
+
.split(",")
|
|
481
|
+
.map((part) => part.trim())
|
|
482
|
+
.filter((part) => part !== "")
|
|
483
|
+
.map((part) => {
|
|
484
|
+
const stem = documentType(part);
|
|
485
|
+
return stem === undefined ? part : DOCUMENT_TYPES[stem];
|
|
486
|
+
});
|
|
487
|
+
}
|
|
488
|
+
const SORTS = {
|
|
489
|
+
relevance: undefined,
|
|
490
|
+
modified: "metadata_modified desc",
|
|
491
|
+
created: "metadata_created desc",
|
|
492
|
+
title: "title_string asc",
|
|
493
|
+
};
|
|
494
|
+
/** CKAN's record types: datasets; Hamburg also documents and apps. */
|
|
495
|
+
export const RECORD_TYPES = ["dataset", "document", "app"];
|
|
496
|
+
/** A facet's items from a search result (`search_facets.<field>.items`), zero counts dropped. */
|
|
497
|
+
function facetItems(result, field) {
|
|
498
|
+
const facets = isObject(result.search_facets) ? result.search_facets : {};
|
|
499
|
+
const facet = facets[field];
|
|
500
|
+
const items = isObject(facet) ? objectsOf(facet["items"]) : [];
|
|
501
|
+
const out = [];
|
|
502
|
+
for (const item of items) {
|
|
503
|
+
const name = typeof item["name"] === "string" ? item["name"] : undefined;
|
|
504
|
+
const count = item["count"];
|
|
505
|
+
if (name === undefined || typeof count !== "number" || count <= 0)
|
|
506
|
+
continue;
|
|
507
|
+
const label = textOf(item["display_name"]);
|
|
508
|
+
out.push(label === undefined || label === name ? { name, count } : { name, label, count });
|
|
509
|
+
}
|
|
510
|
+
return out;
|
|
511
|
+
}
|
|
512
|
+
/** A portal's organizations or groups with their dataset counts (one facet request). */
|
|
513
|
+
async function entriesOf(site, field, trace) {
|
|
514
|
+
const result = await trace.search(site, { rows: 0, facet_field: [field], facet_limit: -1 });
|
|
515
|
+
return facetItems(result, field)
|
|
516
|
+
.map((item) => ({ name: item.name, title: item.label ?? item.name, datasets: item.count }))
|
|
517
|
+
.sort((a, b) => b.datasets - a.datasets || a.name.localeCompare(b.name));
|
|
518
|
+
}
|
|
519
|
+
/** The entries that fit `value`: its slug or title exactly, else those containing it. */
|
|
520
|
+
function matchEntries(entries, value) {
|
|
521
|
+
const folded = foldName(value);
|
|
522
|
+
const exact = entries.filter((o) => o.name.toLowerCase() === folded || foldName(o.title) === folded);
|
|
523
|
+
if (exact.length > 0)
|
|
524
|
+
return { exact: true, matches: exact };
|
|
525
|
+
return { exact: false, matches: entries.filter((o) => o.name.toLowerCase().includes(folded.replace(/ /g, "-")) || foldName(o.title).includes(folded)) };
|
|
526
|
+
}
|
|
527
|
+
const DAY = /^\d{4}-\d{2}-\d{2}$/;
|
|
528
|
+
/** A day (YYYY-MM-DD) checked, as a Date at its start (UTC). */
|
|
529
|
+
function dayOf(name, value) {
|
|
530
|
+
const text = value.trim();
|
|
531
|
+
const date = new Date(`${text}T00:00:00Z`);
|
|
532
|
+
if (!DAY.test(text) || Number.isNaN(date.getTime()) || date.toISOString().slice(0, 10) !== text) {
|
|
533
|
+
throw new CkanValidationError(`Invalid ${name}: expected a day as YYYY-MM-DD, got ${JSON.stringify(text.slice(0, 40))}.`);
|
|
534
|
+
}
|
|
535
|
+
return date;
|
|
536
|
+
}
|
|
537
|
+
/** The Solr range of `field` from the start of `from` to the end of `to` (either open). */
|
|
538
|
+
function dateRange(field, from, to) {
|
|
539
|
+
const start = from === undefined ? "*" : `${from.toISOString().slice(0, 19)}Z`;
|
|
540
|
+
const end = to === undefined ? "*" : `${new Date(to.getTime() + 86_400_000).toISOString().slice(0, 19)}Z`;
|
|
541
|
+
return `${field}:[${start} TO ${end}${to === undefined ? "]" : "}"}`;
|
|
542
|
+
}
|
|
543
|
+
/** The checks of the shared filters (before any request). */
|
|
544
|
+
function checkFilters(filters) {
|
|
545
|
+
const checked = {};
|
|
546
|
+
if (filters.format !== undefined)
|
|
547
|
+
checked.format = wantedFormat(filters.format);
|
|
548
|
+
for (const [name, value] of [["query", filters.query], ["organization", filters.organization], ["group", filters.group], ["tag", filters.tag]]) {
|
|
549
|
+
if (value !== undefined && value.trim() === "")
|
|
550
|
+
throw new CkanValidationError(`Invalid ${name}: expected a non-empty value.`);
|
|
551
|
+
}
|
|
552
|
+
if (filters.type !== undefined && !RECORD_TYPES.includes(filters.type)) {
|
|
553
|
+
throw new CkanValidationError(`Invalid type: expected one of ${RECORD_TYPES.join(", ")}, got ${JSON.stringify(String(filters.type))}.`);
|
|
554
|
+
}
|
|
555
|
+
if (filters.dateField !== undefined && filters.dateField !== "created" && filters.dateField !== "modified") {
|
|
556
|
+
throw new CkanValidationError(`Invalid dateField: expected created or modified, got ${JSON.stringify(String(filters.dateField))}.`);
|
|
557
|
+
}
|
|
558
|
+
if (filters.from !== undefined)
|
|
559
|
+
checked.from = dayOf("from", filters.from);
|
|
560
|
+
if (filters.to !== undefined)
|
|
561
|
+
checked.to = dayOf("to", filters.to);
|
|
562
|
+
if (checked.from !== undefined && checked.to !== undefined && checked.from > checked.to) {
|
|
563
|
+
throw new CkanValidationError("Invalid from/to: from is after to.");
|
|
564
|
+
}
|
|
565
|
+
if (filters.portal !== undefined)
|
|
566
|
+
portalById(filters.portal);
|
|
567
|
+
return checked;
|
|
568
|
+
}
|
|
569
|
+
/** Resolve an organization or group by name against the portal's list; or the answer to give instead. */
|
|
570
|
+
async function resolveEntry(site, field, value, searched, trace) {
|
|
571
|
+
const { exact, matches } = matchEntries(await entriesOf(site, field === "group" ? "groups" : "organization", trace), value);
|
|
572
|
+
if (matches.length === 0) {
|
|
573
|
+
const reason = field === "organization"
|
|
574
|
+
? `No organization of this portal is named ${JSON.stringify(value)}. listOrganizations lists them; the office behind a record is often only in its author or maintainer (search for it as words).`
|
|
575
|
+
: `No group of this portal is named ${JSON.stringify(value)}. listOrganizations with kind group lists them; or search for the theme as words.`;
|
|
576
|
+
return { answer: { status: "empty", searched, reason } };
|
|
577
|
+
}
|
|
578
|
+
if (matches.length > 1) {
|
|
579
|
+
return {
|
|
580
|
+
answer: {
|
|
581
|
+
status: "ambiguous",
|
|
582
|
+
searched,
|
|
583
|
+
reason: `${matches.length} ${field === "organization" ? "organizations" : "groups"} ${exact ? "are named" : "contain"} ${JSON.stringify(value)}: give one of the candidates' names.`,
|
|
584
|
+
ambiguous: { field, value, candidates: matches.slice(0, 10), total: matches.length },
|
|
585
|
+
},
|
|
586
|
+
};
|
|
587
|
+
}
|
|
588
|
+
return matches[0];
|
|
589
|
+
}
|
|
590
|
+
/**
|
|
591
|
+
* The Solr filters for `filters`, an organization or group resolved by name (one request each);
|
|
592
|
+
* or the answer to give instead: empty (no such organization or group) or ambiguous (several).
|
|
593
|
+
*/
|
|
594
|
+
async function filterQueriesOf(site, filters, trace) {
|
|
595
|
+
const checked = checkFilters(filters);
|
|
596
|
+
const searched = {};
|
|
597
|
+
const fq = [];
|
|
598
|
+
if (filters.query !== undefined)
|
|
599
|
+
searched["query"] = filters.query.trim();
|
|
600
|
+
for (const field of ["organization", "group"]) {
|
|
601
|
+
const value = filters[field]?.trim();
|
|
602
|
+
if (value === undefined)
|
|
603
|
+
continue;
|
|
604
|
+
searched[field] = value;
|
|
605
|
+
const entry = await resolveEntry(site, field, value, searched, trace);
|
|
606
|
+
if ("answer" in entry)
|
|
607
|
+
return entry;
|
|
608
|
+
searched[field] = entry.name;
|
|
609
|
+
fq.push(`${field === "group" ? "groups" : "organization"}:${phrase(entry.name)}`);
|
|
610
|
+
}
|
|
611
|
+
if (filters.tag !== undefined) {
|
|
612
|
+
searched["tag"] = filters.tag.trim();
|
|
613
|
+
fq.push(`tags:${phrase(filters.tag.trim())}`);
|
|
614
|
+
}
|
|
615
|
+
if (checked.format !== undefined) {
|
|
616
|
+
searched["format"] = checked.format;
|
|
617
|
+
fq.push(formatFilter(checked.format));
|
|
618
|
+
}
|
|
619
|
+
if (filters.type !== undefined) {
|
|
620
|
+
searched["type"] = filters.type;
|
|
621
|
+
fq.push(`type:${filters.type}`);
|
|
622
|
+
}
|
|
623
|
+
if (filters.withFiles === true) {
|
|
624
|
+
searched["withFiles"] = true;
|
|
625
|
+
fq.push("num_resources:[1 TO *]");
|
|
626
|
+
}
|
|
627
|
+
if (checked.from !== undefined || checked.to !== undefined) {
|
|
628
|
+
const field = filters.dateField ?? "created";
|
|
629
|
+
searched["dateField"] = field;
|
|
630
|
+
if (filters.from !== undefined)
|
|
631
|
+
searched["from"] = filters.from.trim();
|
|
632
|
+
if (filters.to !== undefined)
|
|
633
|
+
searched["to"] = filters.to.trim();
|
|
634
|
+
fq.push(dateRange(`metadata_${field}`, checked.from, checked.to));
|
|
635
|
+
}
|
|
636
|
+
if (filters.filters !== undefined && filters.filters.length > 0) {
|
|
637
|
+
searched["filters"] = [...filters.filters];
|
|
638
|
+
fq.push(...filters.filters);
|
|
639
|
+
}
|
|
640
|
+
return { fq, searched, ...(checked.format === undefined ? {} : { format: checked.format }) };
|
|
641
|
+
}
|
|
642
|
+
/** Throw unless `value` is undefined or an integer in [min, max]. */
|
|
643
|
+
function checkInteger(name, value, min, max) {
|
|
644
|
+
if (value === undefined)
|
|
645
|
+
return;
|
|
646
|
+
if (!Number.isSafeInteger(value) || value < min || value > max) {
|
|
647
|
+
throw new CkanValidationError(`Invalid ${name}: expected an integer from ${min} to ${max}, got ${String(value)}.`);
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
function checkSort(sort) {
|
|
651
|
+
if (!(sort in SORTS))
|
|
652
|
+
throw new CkanValidationError(`Invalid sort: expected one of ${Object.keys(SORTS).join(", ")}, got ${JSON.stringify(String(sort))}.`);
|
|
653
|
+
}
|
|
654
|
+
/** The most results a page may start after: CKAN reads further, but nobody pages that far. */
|
|
655
|
+
export const MAX_OFFSET = 100_000;
|
|
656
|
+
/** The notes every answer on GovData or daten.bw carries: their dataset-level licence is mostly empty. */
|
|
657
|
+
function licenceNotes(site, notes) {
|
|
658
|
+
if (site.portal?.id === "govdata" || site.portal?.id === "bw") {
|
|
659
|
+
notes.push("On this portal the dataset-level licence is mostly empty; the licences listed come from the files where they name one.");
|
|
660
|
+
}
|
|
661
|
+
}
|
|
662
|
+
// ---- findDatasets ----------------------------------------------------------------------------
|
|
663
|
+
export const DEFAULT_DATASETS = 10;
|
|
664
|
+
export const MAX_DATASETS = 50;
|
|
665
|
+
/** A search result as a hit: formats folded, licences named. */
|
|
666
|
+
function hitOf(dataset, format) {
|
|
667
|
+
const resources = objectsOf(dataset["resources"]);
|
|
668
|
+
const extras = extrasOf(dataset);
|
|
669
|
+
const organization = isObject(dataset["organization"]) ? dataset["organization"] : undefined;
|
|
670
|
+
const formats = [...new Set(resources.map((r) => formatOf(r["format"])).filter((f) => f !== undefined))];
|
|
671
|
+
const count = typeof dataset["num_resources"] === "number" ? dataset["num_resources"] : resources.length;
|
|
672
|
+
const hit = {
|
|
673
|
+
name: textOf(dataset["name"]) ?? textOf(dataset["id"]) ?? "",
|
|
674
|
+
files: count,
|
|
675
|
+
formats,
|
|
676
|
+
licences: licencesOf(dataset),
|
|
677
|
+
groups: objectsOf(dataset["groups"]).map((g) => textOf(g["name"])).filter((g) => g !== undefined),
|
|
678
|
+
};
|
|
679
|
+
const set = (key, value) => {
|
|
680
|
+
if (value !== undefined)
|
|
681
|
+
hit[key] = value;
|
|
682
|
+
};
|
|
683
|
+
set("title", textOf(dataset["title"]));
|
|
684
|
+
set("type", textOf(dataset["type"]));
|
|
685
|
+
const orgName = organization === undefined ? undefined : textOf(organization["name"]);
|
|
686
|
+
if (orgName !== undefined) {
|
|
687
|
+
const orgTitle = textOf(organization?.["title"]);
|
|
688
|
+
hit.organization = orgTitle === undefined ? { name: orgName } : { name: orgName, title: orgTitle };
|
|
689
|
+
}
|
|
690
|
+
set("author", textOf(dataset["author"]) ?? textOf(dataset["maintainer"]) ?? extras.get("publisher_name"));
|
|
691
|
+
set("description", plainText(textOf(dataset["notes"]), 300));
|
|
692
|
+
set("modified", textOf(dataset["metadata_modified"]));
|
|
693
|
+
set("created", textOf(dataset["metadata_created"]));
|
|
694
|
+
set("attribution", attributionOf(dataset, extras));
|
|
695
|
+
if (format !== undefined)
|
|
696
|
+
hit.hasFormat = formats.includes(format) || resources.some((r) => typeof r["url"] === "string" && extensionOf(r["url"]) === format);
|
|
697
|
+
return hit;
|
|
698
|
+
}
|
|
699
|
+
/** The notes about a page of hits: metadata-only records, repeated titles, missing licences. */
|
|
700
|
+
function hitNotes(datasets, notes) {
|
|
701
|
+
const stubs = datasets.filter((d) => d.files === 0).length;
|
|
702
|
+
if (stubs > 0)
|
|
703
|
+
notes.push(`${stubs} of these records have no files (metadata only); withFiles leaves them out.`);
|
|
704
|
+
const titles = new Map();
|
|
705
|
+
for (const d of datasets)
|
|
706
|
+
if (d.title !== undefined)
|
|
707
|
+
titles.set(foldName(d.title), (titles.get(foldName(d.title)) ?? 0) + 1);
|
|
708
|
+
const repeated = [...new Set(datasets.filter((d) => d.title !== undefined && (titles.get(foldName(d.title)) ?? 0) > 1).map((d) => d.title))];
|
|
709
|
+
if (repeated.length > 0)
|
|
710
|
+
notes.push(`Titles on this page more than once (editions or series): ${repeated.map((t) => JSON.stringify(t)).join(", ")}. Group them when presenting.`);
|
|
711
|
+
if (datasets.some((d) => d.licences.length === 0))
|
|
712
|
+
notes.push("A record with licences [] states no licence: say so, never assume one.");
|
|
713
|
+
if (datasets.some((d) => d.licences.some((l) => l.open === false)))
|
|
714
|
+
notes.push("Some licences are not open (open false, such as other-closed): say so for those records.");
|
|
715
|
+
}
|
|
716
|
+
/**
|
|
717
|
+
* Datasets for a topic and filters on one portal, a page at a time (`total` counts all matches:
|
|
718
|
+
* read it, never page to count). Each hit names its formats folded, its licences and the
|
|
719
|
+
* attribution text where the record carries one. The notes say which hits are metadata-only, which
|
|
720
|
+
* titles repeat, which licences are not open.
|
|
721
|
+
*/
|
|
722
|
+
export async function findDatasets(client, params = {}) {
|
|
723
|
+
checkInteger("offset", params.offset, 0, MAX_OFFSET);
|
|
724
|
+
checkInteger("limit", params.limit, 1, MAX_DATASETS);
|
|
725
|
+
const sort = params.sort ?? "relevance";
|
|
726
|
+
checkSort(sort);
|
|
727
|
+
const offset = params.offset ?? 0;
|
|
728
|
+
const limit = params.limit ?? DEFAULT_DATASETS;
|
|
729
|
+
checkFilters(params);
|
|
730
|
+
const site = siteOf(client, params.portal);
|
|
731
|
+
const trace = new Trace();
|
|
732
|
+
const filters = await filterQueriesOf(site, params, trace);
|
|
733
|
+
if ("answer" in filters)
|
|
734
|
+
return trace.done({ ...filters.answer, portal: portalRef(site), notes: [], total: 0, offset, datasets: [] });
|
|
735
|
+
const searched = { ...filters.searched, ...(sort === "relevance" ? {} : { sort }) };
|
|
736
|
+
const result = await trace.search(site, { q: params.query?.trim(), fq: filters.fq, rows: limit, start: offset, sort: SORTS[sort] });
|
|
737
|
+
const datasets = result.results.filter(isObject).map((d) => hitOf(d, filters.format));
|
|
738
|
+
const answer = { status: "ok", portal: portalRef(site), total: result.count, offset, datasets, searched, notes: [] };
|
|
739
|
+
if (result.count === 0) {
|
|
740
|
+
answer.status = "empty";
|
|
741
|
+
answer.reason = "No dataset matches on this portal. Try fewer or broader words (Solr matches whole words: Radverkehr, not Rad), fewer filters, or another portal (comparePortals counts them all).";
|
|
742
|
+
}
|
|
743
|
+
else if (datasets.length === 0) {
|
|
744
|
+
answer.status = "empty";
|
|
745
|
+
answer.reason = `The page at offset ${offset} is past the end: ${result.count} datasets match.`;
|
|
746
|
+
}
|
|
747
|
+
if (offset + datasets.length < result.count && datasets.length > 0)
|
|
748
|
+
answer.nextOffset = offset + datasets.length;
|
|
749
|
+
hitNotes(datasets, answer.notes);
|
|
750
|
+
licenceNotes(site, answer.notes);
|
|
751
|
+
return trace.done(answer);
|
|
752
|
+
}
|
|
753
|
+
/** How many portals are asked at once (the server's limiter may allow fewer). */
|
|
754
|
+
const COMPARE_CONCURRENCY = 3;
|
|
755
|
+
/**
|
|
756
|
+
* How many datasets match a query (and a format) on every portal of the built-in list, or on the
|
|
757
|
+
* ones named, most first: one count request per portal, nothing else. A portal that fails is
|
|
758
|
+
* listed with its problem, not left out. GovData, the national catalogue, harvests most Länder
|
|
759
|
+
* portals: it is marked national and must never be added to the others.
|
|
760
|
+
*/
|
|
761
|
+
export async function comparePortals(client, params = {}) {
|
|
762
|
+
if (params.query !== undefined && params.query.trim() === "")
|
|
763
|
+
throw new CkanValidationError("Invalid query: expected a non-empty value.");
|
|
764
|
+
const format = params.format === undefined ? undefined : wantedFormat(params.format);
|
|
765
|
+
const portals = params.portals === undefined || params.portals.length === 0 ? [...PORTALS] : [...new Set(params.portals.map((id) => portalById(id)))];
|
|
766
|
+
const searched = {};
|
|
767
|
+
if (params.query !== undefined)
|
|
768
|
+
searched["query"] = params.query.trim();
|
|
769
|
+
if (format !== undefined)
|
|
770
|
+
searched["format"] = format;
|
|
771
|
+
if (params.portals !== undefined && params.portals.length > 0)
|
|
772
|
+
searched["portals"] = portals.map((p) => p.id);
|
|
773
|
+
const trace = new Trace();
|
|
774
|
+
const search = { q: params.query?.trim(), fq: format === undefined ? undefined : [formatFilter(format)], rows: 0 };
|
|
775
|
+
const sites = portals.map((portal) => siteOf(client, portal.id));
|
|
776
|
+
for (const site of sites)
|
|
777
|
+
trace.requests.push(`${site.prefix}${actionRequest("package_search", packageSearchQuery(search))}`);
|
|
778
|
+
const counts = await mapLimit(sites, COMPARE_CONCURRENCY, async (site) => {
|
|
779
|
+
const portal = site.portal;
|
|
780
|
+
const entry = { id: portal.id, title: portal.title, url: portal.url };
|
|
781
|
+
if (portal.id === "govdata")
|
|
782
|
+
entry.national = true;
|
|
783
|
+
try {
|
|
784
|
+
entry.count = (await site.client.packageSearch(search)).count;
|
|
785
|
+
}
|
|
786
|
+
catch (err) {
|
|
787
|
+
if (!(err instanceof CkanError))
|
|
788
|
+
throw err;
|
|
789
|
+
entry.problem = problemOf(err);
|
|
790
|
+
}
|
|
791
|
+
return entry;
|
|
792
|
+
});
|
|
793
|
+
counts.sort((a, b) => (b.count ?? -1) - (a.count ?? -1) || a.id.localeCompare(b.id));
|
|
794
|
+
const withMatches = counts.filter((c) => (c.count ?? 0) > 0).length;
|
|
795
|
+
const answer = { status: withMatches === 0 ? "empty" : "ok", withMatches, portals: counts, searched, notes: [] };
|
|
796
|
+
if (withMatches === 0)
|
|
797
|
+
answer.reason = "No portal has a matching dataset. Try fewer or broader words (whole words: Radverkehr, not Rad).";
|
|
798
|
+
if (counts.some((c) => c.national === true))
|
|
799
|
+
answer.notes.push("govdata is the national catalogue: it harvests most Länder portals and many cities, so its count overlaps the others. Report it apart and never add it to a sum.");
|
|
800
|
+
if (counts.some((c) => c.id === "hamburg" && (c.count ?? 0) > 0))
|
|
801
|
+
answer.notes.push("hamburg is the Transparenzportal: most of its records are documents (contracts, decisions, reports), not datasets; type dataset narrows it.");
|
|
802
|
+
const failed = counts.filter((c) => c.problem !== undefined);
|
|
803
|
+
if (failed.length > 0)
|
|
804
|
+
answer.notes.push(`${failed.length} portal(s) could not be counted now (${failed.map((c) => `${c.id}: ${c.problem}`).join("; ")}): say so; never read a failure as no data.`);
|
|
805
|
+
return trace.done(answer);
|
|
806
|
+
}
|
|
807
|
+
// ---- countDatasets ---------------------------------------------------------------------------
|
|
808
|
+
/** The distributions countDatasets counts, by name, with CKAN's facet field. */
|
|
809
|
+
export const FACETS = Object.freeze({
|
|
810
|
+
organization: "organization",
|
|
811
|
+
group: "groups",
|
|
812
|
+
format: "res_format",
|
|
813
|
+
licence: "license_id",
|
|
814
|
+
tag: "tags",
|
|
815
|
+
type: "type",
|
|
816
|
+
documentType: "extras_registerobject_type",
|
|
817
|
+
});
|
|
818
|
+
/** Facets whose values are folded (several spellings of one value): all values are read. */
|
|
819
|
+
const FOLDED = new Set(["format", "licence"]);
|
|
820
|
+
export const DEFAULT_FACET_VALUES = 10;
|
|
821
|
+
export const MAX_FACET_VALUES = 100;
|
|
822
|
+
/** The most years one count spreads over (one request each). */
|
|
823
|
+
export const MAX_YEARS = 20;
|
|
824
|
+
const byCount = (a, b) => b.count - a.count || a.value.localeCompare(b.value);
|
|
825
|
+
/** The values of one facet: sorted by count (CKAN doesn't), folded where spellings differ, cut at `limit`. */
|
|
826
|
+
function countsOf(facet, items, limit) {
|
|
827
|
+
if (!FOLDED.has(facet)) {
|
|
828
|
+
const values = items
|
|
829
|
+
.map((item) => {
|
|
830
|
+
const value = { value: item.name, count: item.count };
|
|
831
|
+
const label = facet === "documentType" ? (DOCUMENT_TYPES[item.name] ?? item.label) : item.label;
|
|
832
|
+
if (label !== undefined && label !== item.name)
|
|
833
|
+
value.label = label;
|
|
834
|
+
return value;
|
|
835
|
+
})
|
|
836
|
+
.sort(byCount);
|
|
837
|
+
return { facet, values: values.slice(0, limit), complete: values.length <= limit };
|
|
838
|
+
}
|
|
839
|
+
const groups = new Map();
|
|
840
|
+
const other = { count: 0, variants: [] };
|
|
841
|
+
for (const item of [...items].sort((a, b) => b.count - a.count || a.name.localeCompare(b.name))) {
|
|
842
|
+
const key = facet === "format" ? formatOf(item.name) : licenceOf(item.name, item.label)?.label;
|
|
843
|
+
if (key === undefined) {
|
|
844
|
+
other.count += item.count;
|
|
845
|
+
if (other.variants.length < 10)
|
|
846
|
+
other.variants.push(item.name);
|
|
847
|
+
continue;
|
|
848
|
+
}
|
|
849
|
+
const group = groups.get(key) ?? { value: key, count: 0, variants: [] };
|
|
850
|
+
group.count += item.count;
|
|
851
|
+
group.variants?.push(item.name);
|
|
852
|
+
groups.set(key, group);
|
|
853
|
+
}
|
|
854
|
+
const values = [...groups.values()]
|
|
855
|
+
.map((group) => {
|
|
856
|
+
const variants = group.variants ?? [];
|
|
857
|
+
const value = { value: group.value, count: group.count };
|
|
858
|
+
if (variants.length > 1) {
|
|
859
|
+
value.variants = variants;
|
|
860
|
+
value.upperBound = true;
|
|
861
|
+
}
|
|
862
|
+
else if (variants[0] !== group.value) {
|
|
863
|
+
value.variants = variants;
|
|
864
|
+
}
|
|
865
|
+
return value;
|
|
866
|
+
})
|
|
867
|
+
.sort(byCount);
|
|
868
|
+
const counts = { facet, values: values.slice(0, limit), complete: values.length <= limit };
|
|
869
|
+
if (facet === "format" && other.count > 0)
|
|
870
|
+
counts.other = other;
|
|
871
|
+
return counts;
|
|
872
|
+
}
|
|
873
|
+
/**
|
|
874
|
+
* How many datasets match on one portal, and how they spread over the facets asked for (no
|
|
875
|
+
* datasets fetched), and per year when asked (one count per year on the date field). The values
|
|
876
|
+
* are sorted by count (CKAN sends them unsorted); a format's spellings (CSV, csv, the EU URI) and a
|
|
877
|
+
* licence's (dl-de-by-2.0, dl-by-de/2.0, the URI) are folded into one value, their sum an upper
|
|
878
|
+
* bound; every value of those two is read, so nothing is cut before folding. For the others,
|
|
879
|
+
* complete false says there are more values than listed.
|
|
880
|
+
*/
|
|
881
|
+
export async function countDatasets(client, params = {}) {
|
|
882
|
+
checkInteger("limit", params.limit, 1, MAX_FACET_VALUES);
|
|
883
|
+
checkInteger("yearFrom", params.yearFrom, 1990, 2100);
|
|
884
|
+
checkInteger("yearTo", params.yearTo, 1990, 2100);
|
|
885
|
+
if ((params.yearFrom === undefined) !== (params.yearTo === undefined))
|
|
886
|
+
throw new CkanValidationError("Invalid yearFrom/yearTo: give both, or neither.");
|
|
887
|
+
if (params.yearFrom !== undefined && params.yearTo !== undefined && (params.yearTo < params.yearFrom || params.yearTo - params.yearFrom >= MAX_YEARS)) {
|
|
888
|
+
throw new CkanValidationError(`Invalid yearFrom/yearTo: expected yearFrom ≤ yearTo, at most ${MAX_YEARS} years.`);
|
|
889
|
+
}
|
|
890
|
+
const limit = params.limit ?? DEFAULT_FACET_VALUES;
|
|
891
|
+
const facets = [...new Set(params.facets ?? [])];
|
|
892
|
+
for (const facet of facets) {
|
|
893
|
+
if (!(facet in FACETS))
|
|
894
|
+
throw new CkanValidationError(`Invalid facet: expected one of ${Object.keys(FACETS).join(", ")}, got ${JSON.stringify(String(facet))}.`);
|
|
895
|
+
}
|
|
896
|
+
checkFilters(params);
|
|
897
|
+
const site = siteOf(client, params.portal);
|
|
898
|
+
const trace = new Trace();
|
|
899
|
+
const filters = await filterQueriesOf(site, params, trace);
|
|
900
|
+
if ("answer" in filters)
|
|
901
|
+
return trace.done({ ...filters.answer, portal: portalRef(site), notes: [], total: 0, facets: [] });
|
|
902
|
+
const searched = { ...filters.searched, ...(facets.length > 0 ? { facets } : {}) };
|
|
903
|
+
const base = { q: params.query?.trim(), fq: filters.fq, rows: 0 };
|
|
904
|
+
const folded = facets.filter((f) => FOLDED.has(f));
|
|
905
|
+
const plain = facets.filter((f) => !FOLDED.has(f));
|
|
906
|
+
const results = new Map();
|
|
907
|
+
let total;
|
|
908
|
+
// Folded facets need every value (a cut list can lose one spelling); the others limit + 1, to
|
|
909
|
+
// tell a complete list from a cut one. CKAN takes one facet.limit per request.
|
|
910
|
+
for (const group of [folded, plain]) {
|
|
911
|
+
if (group.length === 0)
|
|
912
|
+
continue;
|
|
913
|
+
const result = await trace.search(site, { ...base, facet_field: group.map((f) => FACETS[f]), facet_limit: group === folded ? -1 : limit + 1 });
|
|
914
|
+
total ??= result.count;
|
|
915
|
+
for (const facet of group)
|
|
916
|
+
results.set(facet, result);
|
|
917
|
+
}
|
|
918
|
+
let byYear;
|
|
919
|
+
if (params.yearFrom !== undefined && params.yearTo !== undefined) {
|
|
920
|
+
const field = `metadata_${params.dateField ?? "created"}`;
|
|
921
|
+
searched["yearFrom"] = params.yearFrom;
|
|
922
|
+
searched["yearTo"] = params.yearTo;
|
|
923
|
+
searched["dateField"] = params.dateField ?? "created";
|
|
924
|
+
byYear = [];
|
|
925
|
+
for (let year = params.yearFrom; year <= params.yearTo; year++) {
|
|
926
|
+
const range = `${field}:[${year}-01-01T00:00:00Z TO ${year + 1}-01-01T00:00:00Z}`;
|
|
927
|
+
const result = await trace.search(site, { ...base, fq: [...filters.fq, range] });
|
|
928
|
+
byYear.push({ year, count: result.count });
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
if (total === undefined)
|
|
932
|
+
total = (await trace.search(site, base)).count;
|
|
933
|
+
const answer = {
|
|
934
|
+
status: total === 0 ? "empty" : "ok",
|
|
935
|
+
portal: portalRef(site),
|
|
936
|
+
total,
|
|
937
|
+
facets: facets.map((facet) => countsOf(facet, facetItems(results.get(facet), FACETS[facet]), limit)),
|
|
938
|
+
searched,
|
|
939
|
+
notes: [],
|
|
940
|
+
};
|
|
941
|
+
if (byYear !== undefined)
|
|
942
|
+
answer.byYear = byYear;
|
|
943
|
+
if (total === 0)
|
|
944
|
+
answer.reason = "No dataset matches on this portal. Try fewer or broader words, or fewer filters.";
|
|
945
|
+
if (facets.length > 0 && total > 0)
|
|
946
|
+
answer.notes.push("Counts are datasets, not files. One dataset can carry several values, so the counts don't add up to total.");
|
|
947
|
+
if (answer.facets.some((f) => f.values.some((v) => v.upperBound === true))) {
|
|
948
|
+
answer.notes.push("A value with upperBound sums several spellings (CSV and csv, dl-de-by-2.0 and its URI); a dataset with two of them counts twice. For an exact number, count again with format set: total is exact.");
|
|
949
|
+
}
|
|
950
|
+
if (facets.includes("tag") && site.portal?.id === "hamburg")
|
|
951
|
+
answer.notes.push("Hamburg's tags are often whole keyword lists in one string: a tag ranking there says little.");
|
|
952
|
+
if (facets.includes("documentType")) {
|
|
953
|
+
answer.notes.push(site.portal?.id === "hamburg" ? "A record can carry several document types, so their counts overlap." : "documentType is the Transparenzportal Hamburg's field; other portals have none.");
|
|
954
|
+
}
|
|
955
|
+
if (facets.includes("licence"))
|
|
956
|
+
licenceNotes(site, answer.notes);
|
|
957
|
+
if (byYear !== undefined) {
|
|
958
|
+
const when = params.dateField === "modified" ? "when the record last changed on the portal" : "when the record entered the portal";
|
|
959
|
+
answer.notes.push(`byYear counts by ${params.dateField ?? "created"} (${when}), not by the date of the data.`);
|
|
960
|
+
}
|
|
961
|
+
return trace.done(answer);
|
|
962
|
+
}
|
|
963
|
+
// ---- findHamburgDocuments --------------------------------------------------------------------
|
|
964
|
+
export const DEFAULT_DOCUMENTS = 10;
|
|
965
|
+
export const MAX_DOCUMENTS = 50;
|
|
966
|
+
/** The most files listed per document. */
|
|
967
|
+
const MAX_FILES_PER_DOCUMENT = 10;
|
|
968
|
+
/** A search hit as a Transparenzportal document. */
|
|
969
|
+
function documentOf(dataset) {
|
|
970
|
+
const extras = extrasOf(dataset);
|
|
971
|
+
const files = objectsOf(dataset["resources"]).map(fileOf).filter((f) => f !== undefined);
|
|
972
|
+
const doc = {
|
|
973
|
+
name: textOf(dataset["name"]) ?? textOf(dataset["id"]) ?? "",
|
|
974
|
+
documentTypes: documentTypesOf(extras.get("registerobject_type")),
|
|
975
|
+
counts: {
|
|
976
|
+
files: files.length,
|
|
977
|
+
pdfs: files.filter((f) => f.format === "PDF" || f.extension === "PDF").length,
|
|
978
|
+
knownBytes: files.reduce((sum, f) => sum + (f.size ?? 0), 0),
|
|
979
|
+
},
|
|
980
|
+
files: files.slice(0, MAX_FILES_PER_DOCUMENT).map(({ id: _id, ...file }) => file),
|
|
981
|
+
};
|
|
982
|
+
const set = (key, value) => {
|
|
983
|
+
if (value !== undefined)
|
|
984
|
+
doc[key] = value;
|
|
985
|
+
};
|
|
986
|
+
set("title", textOf(dataset["title"]));
|
|
987
|
+
set("published", extras.get("publishing_date"));
|
|
988
|
+
set("created", textOf(dataset["metadata_created"]));
|
|
989
|
+
set("offline", extras.get("offline_date"));
|
|
990
|
+
set("organization", isObject(dataset["organization"]) ? textOf(dataset["organization"]["name"]) : undefined);
|
|
991
|
+
set("author", textOf(dataset["author"]));
|
|
992
|
+
set("description", plainText(textOf(dataset["notes"]), 300));
|
|
993
|
+
set("fileReference", extras.get("file_reference_digital"));
|
|
994
|
+
set("licence", licenceOf(dataset["license_id"], dataset["license_title"]));
|
|
995
|
+
set("attribution", attributionOf(dataset, extras));
|
|
996
|
+
return doc;
|
|
997
|
+
}
|
|
998
|
+
/**
|
|
999
|
+
* Documents of the Transparenzportal Hamburg (the register the Hamburg Transparency Act requires):
|
|
1000
|
+
* by document type (sent as the stemmed value its search index holds, the only one that works),
|
|
1001
|
+
* topic and the day they entered the portal, newest first, a page at a time. Each with its
|
|
1002
|
+
* publication date, types, office, licence and attribution text, and its files. Always on Hamburg's
|
|
1003
|
+
* portal, whatever portal the client talks to.
|
|
1004
|
+
*/
|
|
1005
|
+
export async function findHamburgDocuments(client, params = {}) {
|
|
1006
|
+
checkInteger("offset", params.offset, 0, MAX_OFFSET);
|
|
1007
|
+
checkInteger("limit", params.limit, 1, MAX_DOCUMENTS);
|
|
1008
|
+
const sort = params.sort ?? "created";
|
|
1009
|
+
checkSort(sort);
|
|
1010
|
+
let type;
|
|
1011
|
+
if (params.documentType !== undefined) {
|
|
1012
|
+
type = documentType(params.documentType);
|
|
1013
|
+
if (type === undefined) {
|
|
1014
|
+
throw new CkanValidationError(`Invalid documentType: ${JSON.stringify(params.documentType.slice(0, 80))} is none of Hamburg's document types (${Object.values(DOCUMENT_TYPES).join(", ")}).`);
|
|
1015
|
+
}
|
|
1016
|
+
}
|
|
1017
|
+
if (params.organization !== undefined && !/^[a-z0-9_-]{2,100}$/.test(params.organization.trim())) {
|
|
1018
|
+
throw new CkanValidationError("Invalid organization: expected a slug such as allris, workflows, hmdklgv, dokrates, statistikamt or stadthamburg.");
|
|
1019
|
+
}
|
|
1020
|
+
const offset = params.offset ?? 0;
|
|
1021
|
+
const limit = params.limit ?? DEFAULT_DOCUMENTS;
|
|
1022
|
+
const filters = { query: params.query, from: params.from, to: params.to, dateField: "created" };
|
|
1023
|
+
const checked = checkFilters(filters);
|
|
1024
|
+
const site = siteOf(client, "hamburg");
|
|
1025
|
+
const trace = new Trace();
|
|
1026
|
+
const searched = {};
|
|
1027
|
+
const fq = [];
|
|
1028
|
+
if (type !== undefined) {
|
|
1029
|
+
searched["documentType"] = DOCUMENT_TYPES[type];
|
|
1030
|
+
fq.push(`extras_registerobject_type:${type}`);
|
|
1031
|
+
}
|
|
1032
|
+
if (params.query !== undefined)
|
|
1033
|
+
searched["query"] = params.query.trim();
|
|
1034
|
+
if (params.organization !== undefined) {
|
|
1035
|
+
searched["organization"] = params.organization.trim();
|
|
1036
|
+
fq.push(`organization:${params.organization.trim()}`);
|
|
1037
|
+
}
|
|
1038
|
+
if (checked.from !== undefined || checked.to !== undefined) {
|
|
1039
|
+
if (params.from !== undefined)
|
|
1040
|
+
searched["from"] = params.from.trim();
|
|
1041
|
+
if (params.to !== undefined)
|
|
1042
|
+
searched["to"] = params.to.trim();
|
|
1043
|
+
fq.push(dateRange("metadata_created", checked.from, checked.to));
|
|
1044
|
+
}
|
|
1045
|
+
if (sort !== "relevance")
|
|
1046
|
+
searched["sort"] = sort;
|
|
1047
|
+
const result = await trace.search(site, { q: params.query?.trim(), fq, rows: limit, start: offset, sort: SORTS[sort] });
|
|
1048
|
+
const documents = result.results.filter(isObject).map(documentOf);
|
|
1049
|
+
const answer = { status: "ok", portal: portalRef(site), total: result.count, offset, documents, searched, notes: [] };
|
|
1050
|
+
if (result.count === 0) {
|
|
1051
|
+
answer.status = "empty";
|
|
1052
|
+
answer.reason = "No document matches. Try fewer or broader words, another document type, or a wider date range.";
|
|
1053
|
+
}
|
|
1054
|
+
else if (documents.length === 0) {
|
|
1055
|
+
answer.status = "empty";
|
|
1056
|
+
answer.reason = `The page at offset ${offset} is past the end: ${result.count} documents match.`;
|
|
1057
|
+
}
|
|
1058
|
+
if (offset + documents.length < result.count && documents.length > 0)
|
|
1059
|
+
answer.nextOffset = offset + documents.length;
|
|
1060
|
+
if (params.organization !== undefined)
|
|
1061
|
+
answer.notes.push("Many records have no organization (only about half the contracts and a third of the decisions have one): the organization filter drops the rest. Filter by type and words instead.");
|
|
1062
|
+
if (documents.some((d) => d.documentTypes.length > 1))
|
|
1063
|
+
answer.notes.push("A record can carry several document types, so type counts overlap.");
|
|
1064
|
+
answer.notes.push("Licences are set per record (most DL-DE BY 2.0, which asks for the attribution text; some CC0): give each document's own, and quote its attribution when it is reused.");
|
|
1065
|
+
return trace.done(answer);
|
|
1066
|
+
}
|
|
1067
|
+
// ---- getDataset ------------------------------------------------------------------------------
|
|
1068
|
+
/** A dataset's slug or id from itself or a link to its page (…/dataset/NAME). */
|
|
1069
|
+
export function datasetIdFrom(value) {
|
|
1070
|
+
const text = value.trim();
|
|
1071
|
+
const link = /^https?:\/\/[^/]+\/(?:[^?#]*\/)?(?:dataset|daten)\/([^/?#]+)\/?(?:[?#].*)?$/i.exec(text);
|
|
1072
|
+
if (link === null)
|
|
1073
|
+
return text;
|
|
1074
|
+
try {
|
|
1075
|
+
return decodeURIComponent(link[1]);
|
|
1076
|
+
}
|
|
1077
|
+
catch {
|
|
1078
|
+
return link[1];
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
/** Whether CKAN answered "no such dataset" (HTTP 404, or `Not Found Error` with HTTP 200). */
|
|
1082
|
+
function isNotFound(err) {
|
|
1083
|
+
return (err instanceof CkanApiError && err.status === 404) || (err instanceof CkanActionError && err.errorType === "Not Found Error");
|
|
1084
|
+
}
|
|
1085
|
+
/** The extras getDataset names, by their key, under the name it gives them. */
|
|
1086
|
+
const DETAIL_KEYS = [
|
|
1087
|
+
["publishing_date", "published"],
|
|
1088
|
+
["offline_date", "offline"],
|
|
1089
|
+
["temporal_coverage_from", "temporalFrom"],
|
|
1090
|
+
["temporal_coverage_to", "temporalTo"],
|
|
1091
|
+
["temporal_start", "temporalFrom"],
|
|
1092
|
+
["temporal_end", "temporalTo"],
|
|
1093
|
+
["file_reference_digital", "fileReference"],
|
|
1094
|
+
["publisher_name", "publisher"],
|
|
1095
|
+
["opennrw_datapublisher", "publisher"],
|
|
1096
|
+
["issued", "issued"],
|
|
1097
|
+
["modified", "sourceModified"],
|
|
1098
|
+
["metadata_harvested_portal", "harvestedFrom"],
|
|
1099
|
+
["metadata_original_portal", "originalPortal"],
|
|
1100
|
+
["contact_email", "contactEmail"],
|
|
1101
|
+
];
|
|
1102
|
+
/**
|
|
1103
|
+
* One record by its name (slug), id or the link to its page, on one portal: organization, author,
|
|
1104
|
+
* groups, tags, dates, licences and attribution, selected details, and every file with its kind,
|
|
1105
|
+
* format, size and licence. An unknown name is an empty answer that says where to take one from.
|
|
1106
|
+
*/
|
|
1107
|
+
export async function getDataset(client, params) {
|
|
1108
|
+
if (typeof params.id !== "string" || params.id.trim() === "")
|
|
1109
|
+
throw new CkanValidationError("Invalid id: expected a dataset's name, id or the link to its page.");
|
|
1110
|
+
const id = datasetIdFrom(params.id);
|
|
1111
|
+
const site = siteOf(client, params.portal);
|
|
1112
|
+
const trace = new Trace();
|
|
1113
|
+
const searched = { id };
|
|
1114
|
+
trace.requests.push(`${site.prefix}${actionRequest("package_show", { id })}`);
|
|
1115
|
+
let dataset;
|
|
1116
|
+
try {
|
|
1117
|
+
dataset = await site.client.packageShow(id);
|
|
1118
|
+
}
|
|
1119
|
+
catch (err) {
|
|
1120
|
+
if (!isNotFound(err))
|
|
1121
|
+
throw err;
|
|
1122
|
+
return trace.done({ status: "empty", portal: portalRef(site), searched, notes: [], reason: `No record is named ${JSON.stringify(id)} on this portal. Take the name (the slug) from a fresh findDatasets answer on the same portal; titles are not names.` });
|
|
1123
|
+
}
|
|
1124
|
+
const extras = extrasOf(dataset);
|
|
1125
|
+
const organization = isObject(dataset["organization"]) ? dataset["organization"] : undefined;
|
|
1126
|
+
const files = objectsOf(dataset["resources"]).map(fileOf).filter((f) => f !== undefined);
|
|
1127
|
+
const details = {};
|
|
1128
|
+
for (const [key, name] of DETAIL_KEYS) {
|
|
1129
|
+
const value = extras.get(key);
|
|
1130
|
+
if (value !== undefined && details[name] === undefined)
|
|
1131
|
+
details[name] = excerpt(value, 300);
|
|
1132
|
+
}
|
|
1133
|
+
const types = documentTypesOf(extras.get("registerobject_type"));
|
|
1134
|
+
if (types.length > 0)
|
|
1135
|
+
details["documentTypes"] = types;
|
|
1136
|
+
const record = {
|
|
1137
|
+
name: textOf(dataset["name"]) ?? id,
|
|
1138
|
+
groups: objectsOf(dataset["groups"]).flatMap((g) => {
|
|
1139
|
+
const name = textOf(g["name"]);
|
|
1140
|
+
if (name === undefined)
|
|
1141
|
+
return [];
|
|
1142
|
+
const title = textOf(g["title"]) ?? textOf(g["display_name"]);
|
|
1143
|
+
return [title === undefined ? { name } : { name, title }];
|
|
1144
|
+
}),
|
|
1145
|
+
tags: objectsOf(dataset["tags"]).map((t) => textOf(t["name"])).filter((t) => t !== undefined),
|
|
1146
|
+
licences: licencesOf(dataset),
|
|
1147
|
+
details,
|
|
1148
|
+
files,
|
|
1149
|
+
};
|
|
1150
|
+
const set = (key, value) => {
|
|
1151
|
+
if (value !== undefined)
|
|
1152
|
+
record[key] = value;
|
|
1153
|
+
};
|
|
1154
|
+
set("id", textOf(dataset["id"]));
|
|
1155
|
+
set("title", textOf(dataset["title"]));
|
|
1156
|
+
set("type", textOf(dataset["type"]));
|
|
1157
|
+
set("description", plainText(textOf(dataset["notes"]), 5000));
|
|
1158
|
+
const orgName = organization === undefined ? undefined : textOf(organization["name"]);
|
|
1159
|
+
if (orgName !== undefined) {
|
|
1160
|
+
const orgTitle = textOf(organization?.["title"]);
|
|
1161
|
+
record.organization = orgTitle === undefined ? { name: orgName } : { name: orgName, title: orgTitle };
|
|
1162
|
+
}
|
|
1163
|
+
set("author", textOf(dataset["author"]));
|
|
1164
|
+
set("maintainer", textOf(dataset["maintainer"]));
|
|
1165
|
+
set("modified", textOf(dataset["metadata_modified"]));
|
|
1166
|
+
set("created", textOf(dataset["metadata_created"]));
|
|
1167
|
+
set("landingPage", textOf(dataset["url"]) ?? extras.get("metadata_original_html"));
|
|
1168
|
+
set("attribution", attributionOf(dataset, extras));
|
|
1169
|
+
const answer = { status: "ok", portal: portalRef(site), dataset: record, searched, notes: [] };
|
|
1170
|
+
if (files.length === 0)
|
|
1171
|
+
answer.notes.push("This record has no files with a link (metadata only): see landingPage.");
|
|
1172
|
+
const odd = [...new Set(files.filter((f) => !FETCHABLE.has(f.kind)).map((f) => f.kind))];
|
|
1173
|
+
if (odd.length > 0)
|
|
1174
|
+
answer.notes.push(`Not every link is a file (${odd.join(", ")}): only kind file and generated can be fetched; name the others as what they are.`);
|
|
1175
|
+
if (record.licences.length === 0)
|
|
1176
|
+
answer.notes.push("No licence is stated: say so, never assume one; the publisher's page may.");
|
|
1177
|
+
else if (record.licences.length > 1)
|
|
1178
|
+
answer.notes.push("The record and its files name different licences: give each file's own.");
|
|
1179
|
+
if (record.licences.some((l) => l.open === false))
|
|
1180
|
+
answer.notes.push("A licence here is not open (open false): say so.");
|
|
1181
|
+
licenceNotes(site, answer.notes);
|
|
1182
|
+
return trace.done(answer);
|
|
1183
|
+
}
|
|
1184
|
+
/**
|
|
1185
|
+
* A portal's organizations (publishers), or its groups (themes), most datasets first, with their
|
|
1186
|
+
* slugs for the filters; with a name, those whose slug or title contains it. Counted from the
|
|
1187
|
+
* search's facet, so only those with datasets.
|
|
1188
|
+
*/
|
|
1189
|
+
export async function listOrganizations(client, params = {}) {
|
|
1190
|
+
if (params.name !== undefined && params.name.trim() === "")
|
|
1191
|
+
throw new CkanValidationError("Invalid name: expected a non-empty value.");
|
|
1192
|
+
const kind = params.kind ?? "organization";
|
|
1193
|
+
if (kind !== "organization" && kind !== "group")
|
|
1194
|
+
throw new CkanValidationError(`Invalid kind: expected organization or group, got ${JSON.stringify(String(kind))}.`);
|
|
1195
|
+
const site = siteOf(client, params.portal);
|
|
1196
|
+
const trace = new Trace();
|
|
1197
|
+
const all = await entriesOf(site, kind === "group" ? "groups" : "organization", trace);
|
|
1198
|
+
const searched = { kind, ...(params.name === undefined ? {} : { name: params.name.trim() }) };
|
|
1199
|
+
const organizations = params.name === undefined ? all : matchEntries(all, params.name).matches;
|
|
1200
|
+
const answer = { status: "ok", portal: portalRef(site), kind, total: all.length, organizations, searched, notes: [] };
|
|
1201
|
+
if (organizations.length === 0) {
|
|
1202
|
+
answer.status = "empty";
|
|
1203
|
+
answer.reason =
|
|
1204
|
+
all.length === 0
|
|
1205
|
+
? `This portal lists no ${kind === "group" ? "groups" : "organizations"} with datasets.`
|
|
1206
|
+
: `No ${kind} is named or contains ${JSON.stringify(params.name?.trim())}. The office behind a record is often only its author: search for it as words.`;
|
|
1207
|
+
}
|
|
1208
|
+
if (kind === "organization" && site.portal?.id === "hamburg")
|
|
1209
|
+
answer.notes.push("On Hamburg many records have no organization at all (the office is the author): the counts cover only those that have one.");
|
|
1210
|
+
return trace.done(answer);
|
|
1211
|
+
}
|