@usa-open-data-connectors/usa-sources 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +229 -32
- package/dist/btsTransportationStats.d.ts +4 -0
- package/dist/btsTransportationStats.d.ts.map +1 -1
- package/dist/btsTransportationStats.js +3 -0
- package/dist/btsTransportationStats.js.map +1 -1
- package/dist/californiaOpenDataCatalogue.d.ts +84 -0
- package/dist/californiaOpenDataCatalogue.d.ts.map +1 -0
- package/dist/californiaOpenDataCatalogue.js +153 -0
- package/dist/californiaOpenDataCatalogue.js.map +1 -0
- package/dist/cdcSocrataCatalogue.d.ts +4 -0
- package/dist/cdcSocrataCatalogue.d.ts.map +1 -1
- package/dist/cdcSocrataCatalogue.js +3 -0
- package/dist/cdcSocrataCatalogue.js.map +1 -1
- package/dist/chicagoOpenDataCatalogue.d.ts +91 -0
- package/dist/chicagoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/chicagoOpenDataCatalogue.js +133 -0
- package/dist/chicagoOpenDataCatalogue.js.map +1 -0
- package/dist/coloradoOpenDataCatalogue.d.ts +42 -0
- package/dist/coloradoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/coloradoOpenDataCatalogue.js +45 -0
- package/dist/coloradoOpenDataCatalogue.js.map +1 -0
- package/dist/fccSocrataCatalogue.d.ts +91 -0
- package/dist/fccSocrataCatalogue.d.ts.map +1 -0
- package/dist/fccSocrataCatalogue.js +133 -0
- package/dist/fccSocrataCatalogue.js.map +1 -0
- package/dist/federalRegisterDocuments.d.ts +78 -0
- package/dist/federalRegisterDocuments.d.ts.map +1 -0
- package/dist/federalRegisterDocuments.js +119 -0
- package/dist/federalRegisterDocuments.js.map +1 -0
- package/dist/healthdataSocrataCatalogue.d.ts +4 -0
- package/dist/healthdataSocrataCatalogue.d.ts.map +1 -1
- package/dist/healthdataSocrataCatalogue.js +3 -0
- package/dist/healthdataSocrataCatalogue.js.map +1 -1
- package/dist/http.d.ts +14 -0
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +29 -1
- package/dist/http.js.map +1 -1
- package/dist/illinoisOpenDataCatalogue.d.ts +42 -0
- package/dist/illinoisOpenDataCatalogue.d.ts.map +1 -0
- package/dist/illinoisOpenDataCatalogue.js +45 -0
- package/dist/illinoisOpenDataCatalogue.js.map +1 -0
- package/dist/index.d.ts +76 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -0
- package/dist/index.js.map +1 -1
- package/dist/losAngelesOpenDataCatalogue.d.ts +42 -0
- package/dist/losAngelesOpenDataCatalogue.d.ts.map +1 -0
- package/dist/losAngelesOpenDataCatalogue.js +48 -0
- package/dist/losAngelesOpenDataCatalogue.js.map +1 -0
- package/dist/michiganOpenDataCatalogue.d.ts +42 -0
- package/dist/michiganOpenDataCatalogue.d.ts.map +1 -0
- package/dist/michiganOpenDataCatalogue.js +45 -0
- package/dist/michiganOpenDataCatalogue.js.map +1 -0
- package/dist/newJerseyOpenDataCatalogue.d.ts +42 -0
- package/dist/newJerseyOpenDataCatalogue.d.ts.map +1 -0
- package/dist/newJerseyOpenDataCatalogue.js +48 -0
- package/dist/newJerseyOpenDataCatalogue.js.map +1 -0
- package/dist/nextBusTransitVehicles.d.ts.map +1 -1
- package/dist/nextBusTransitVehicles.js +14 -3
- package/dist/nextBusTransitVehicles.js.map +1 -1
- package/dist/nhtsaVehicleModels.d.ts +74 -0
- package/dist/nhtsaVehicleModels.d.ts.map +1 -0
- package/dist/nhtsaVehicleModels.js +139 -0
- package/dist/nhtsaVehicleModels.js.map +1 -0
- package/dist/nihReporterProjects.d.ts +98 -0
- package/dist/nihReporterProjects.d.ts.map +1 -0
- package/dist/nihReporterProjects.js +188 -0
- package/dist/nihReporterProjects.js.map +1 -0
- package/dist/nwsPointForecast.d.ts.map +1 -1
- package/dist/nwsPointForecast.js +13 -2
- package/dist/nwsPointForecast.js.map +1 -1
- package/dist/nyStateOpenDataCatalogue.d.ts +91 -0
- package/dist/nyStateOpenDataCatalogue.d.ts.map +1 -0
- package/dist/nyStateOpenDataCatalogue.js +133 -0
- package/dist/nyStateOpenDataCatalogue.js.map +1 -0
- package/dist/nycOpenDataCatalogue.d.ts +4 -0
- package/dist/nycOpenDataCatalogue.d.ts.map +1 -1
- package/dist/nycOpenDataCatalogue.js +3 -0
- package/dist/nycOpenDataCatalogue.js.map +1 -1
- package/dist/oregonOpenDataCatalogue.d.ts +42 -0
- package/dist/oregonOpenDataCatalogue.d.ts.map +1 -0
- package/dist/oregonOpenDataCatalogue.js +45 -0
- package/dist/oregonOpenDataCatalogue.js.map +1 -0
- package/dist/pennsylvaniaOpenDataCatalogue.d.ts +42 -0
- package/dist/pennsylvaniaOpenDataCatalogue.d.ts.map +1 -0
- package/dist/pennsylvaniaOpenDataCatalogue.js +48 -0
- package/dist/pennsylvaniaOpenDataCatalogue.js.map +1 -0
- package/dist/registry.d.ts +2 -1
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +51 -3
- package/dist/registry.js.map +1 -1
- package/dist/sanFranciscoOpenDataCatalogue.d.ts +42 -0
- package/dist/sanFranciscoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/sanFranciscoOpenDataCatalogue.js +48 -0
- package/dist/sanFranciscoOpenDataCatalogue.js.map +1 -0
- package/dist/seattleOpenDataCatalogue.d.ts +91 -0
- package/dist/seattleOpenDataCatalogue.d.ts.map +1 -0
- package/dist/seattleOpenDataCatalogue.js +133 -0
- package/dist/seattleOpenDataCatalogue.js.map +1 -0
- package/dist/socrataCatalogue.d.ts +121 -0
- package/dist/socrataCatalogue.d.ts.map +1 -0
- package/dist/socrataCatalogue.js +151 -0
- package/dist/socrataCatalogue.js.map +1 -0
- package/dist/socrataCatalogueMetadata.d.ts +19 -0
- package/dist/socrataCatalogueMetadata.d.ts.map +1 -0
- package/dist/socrataCatalogueMetadata.js +40 -0
- package/dist/socrataCatalogueMetadata.js.map +1 -0
- package/dist/texasOpenDataCatalogue.d.ts +91 -0
- package/dist/texasOpenDataCatalogue.d.ts.map +1 -0
- package/dist/texasOpenDataCatalogue.js +133 -0
- package/dist/texasOpenDataCatalogue.js.map +1 -0
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/vermontOpenDataCatalogue.d.ts +42 -0
- package/dist/vermontOpenDataCatalogue.d.ts.map +1 -0
- package/dist/vermontOpenDataCatalogue.js +45 -0
- package/dist/vermontOpenDataCatalogue.js.map +1 -0
- package/dist/washingtonStateOpenDataCatalogue.d.ts +42 -0
- package/dist/washingtonStateOpenDataCatalogue.d.ts.map +1 -0
- package/dist/washingtonStateOpenDataCatalogue.js +48 -0
- package/dist/washingtonStateOpenDataCatalogue.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { UsSourceParseError } from './errors.js';
|
|
3
|
+
import { readFixtureJson } from './fixtures.js';
|
|
4
|
+
import { httpGet } from './http.js';
|
|
5
|
+
/** Adapter id for the California Open Data catalogue source. */
|
|
6
|
+
export const CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID = 'california-open-data-catalogue';
|
|
7
|
+
/** Keyless CKAN package-search URL for the California Open Data catalogue. */
|
|
8
|
+
export const CALIFORNIA_OPEN_DATA_API_BASE = 'https://data.ca.gov/api/3/action/package_search';
|
|
9
|
+
/** How many catalogue entries one request reads. */
|
|
10
|
+
export const CALIFORNIA_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
11
|
+
/** Committed snapshot, so a build works when the California host is unreachable. */
|
|
12
|
+
const CALIFORNIA_OPEN_DATA_FIXTURE_FILENAME = 'california-open-data-catalogue-2026-11-08.json';
|
|
13
|
+
/** Category bucket the adapter uses for an entry the catalogue lists without one. */
|
|
14
|
+
const CALIFORNIA_OPEN_DATA_UNCATEGORISED = 'Uncategorised';
|
|
15
|
+
const CALIFORNIA_OPEN_DATA_PACKAGE_SCHEMA = z.object({
|
|
16
|
+
id: z.string().optional(),
|
|
17
|
+
name: z.string().optional(),
|
|
18
|
+
title: z.string().optional(),
|
|
19
|
+
type: z.string().optional(),
|
|
20
|
+
metadata_created: z.string().optional(),
|
|
21
|
+
metadata_modified: z.string().optional(),
|
|
22
|
+
license_url: z.string().nullable().optional(),
|
|
23
|
+
license_title: z.string().nullable().optional(),
|
|
24
|
+
organization: z.object({ title: z.string().optional() }).nullable().optional(),
|
|
25
|
+
groups: z.array(z.object({ title: z.string().optional() })).optional(),
|
|
26
|
+
num_resources: z.number().optional(),
|
|
27
|
+
resources: z.array(z.unknown()).optional(),
|
|
28
|
+
});
|
|
29
|
+
const CALIFORNIA_OPEN_DATA_PAYLOAD_SCHEMA = z.object({
|
|
30
|
+
success: z.literal(true),
|
|
31
|
+
result: z.object({
|
|
32
|
+
count: z.number(),
|
|
33
|
+
results: z.array(CALIFORNIA_OPEN_DATA_PACKAGE_SCHEMA),
|
|
34
|
+
}),
|
|
35
|
+
});
|
|
36
|
+
/** Reads one CKAN timestamp as seconds since the Unix epoch. */
|
|
37
|
+
function parseCkanTimestamp(sourceId, value, field) {
|
|
38
|
+
const normalized = /(?:Z|[+-]\d{2}:?\d{2})$/.test(value) ? value : `${value}Z`;
|
|
39
|
+
const millis = Date.parse(normalized);
|
|
40
|
+
if (!Number.isFinite(millis)) {
|
|
41
|
+
throw new UsSourceParseError(sourceId, `A catalogue entry carried an invalid ${field}`);
|
|
42
|
+
}
|
|
43
|
+
return Math.floor(millis / 1000);
|
|
44
|
+
}
|
|
45
|
+
/** Counts entries per category, most entries first, ties by name. */
|
|
46
|
+
function countCaliforniaOpenDataCategories(entries) {
|
|
47
|
+
const counts = new Map();
|
|
48
|
+
for (const entry of entries) {
|
|
49
|
+
const category = entry.category ?? CALIFORNIA_OPEN_DATA_UNCATEGORISED;
|
|
50
|
+
counts.set(category, (counts.get(category) ?? 0) + 1);
|
|
51
|
+
}
|
|
52
|
+
return [...counts.entries()]
|
|
53
|
+
.sort(([leftCategory, leftCount], [rightCategory, rightCount]) => leftCount === rightCount ? leftCategory.localeCompare(rightCategory) : rightCount - leftCount)
|
|
54
|
+
.map(([category, entryCount]) => ({ category, entryCount }));
|
|
55
|
+
}
|
|
56
|
+
/** Picks the entry with the latest creation time, ties to the earlier entry. */
|
|
57
|
+
function newestCaliforniaOpenDataEntry(entries) {
|
|
58
|
+
const first = entries[0];
|
|
59
|
+
if (first === undefined) {
|
|
60
|
+
throw new UsSourceParseError(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, 'No entries to summarise');
|
|
61
|
+
}
|
|
62
|
+
return entries.reduce((best, entry) => (entry.createdAtEpochSeconds > best.createdAtEpochSeconds ? entry : best), first);
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Builds the request URL for one page of the California CKAN catalogue.
|
|
66
|
+
*
|
|
67
|
+
* The endpoint is keyless. CKAN calls the entry cap `rows`.
|
|
68
|
+
*
|
|
69
|
+
* @param options - an optional entry cap
|
|
70
|
+
* @returns the full request URL
|
|
71
|
+
*/
|
|
72
|
+
export function buildCaliforniaOpenDataCatalogueUrl(options) {
|
|
73
|
+
const params = new URLSearchParams({
|
|
74
|
+
rows: String(options?.limit ?? CALIFORNIA_OPEN_DATA_CATALOGUE_LIMIT),
|
|
75
|
+
});
|
|
76
|
+
return `${CALIFORNIA_OPEN_DATA_API_BASE}?${params.toString()}`;
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Parses one page of the California CKAN catalogue into entries and topic counts.
|
|
80
|
+
*
|
|
81
|
+
* An entry without an id, a name, a title, or a creation time is a changed
|
|
82
|
+
* shape and stops the parse. An empty page also stops the parse.
|
|
83
|
+
*
|
|
84
|
+
* @param payload - the JSON body from the California CKAN package search
|
|
85
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
86
|
+
*/
|
|
87
|
+
export function parseCaliforniaOpenDataCataloguePayload(payload) {
|
|
88
|
+
const parsed = CALIFORNIA_OPEN_DATA_PAYLOAD_SCHEMA.safeParse(payload);
|
|
89
|
+
if (!parsed.success) {
|
|
90
|
+
throw new UsSourceParseError(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, parsed.error.message);
|
|
91
|
+
}
|
|
92
|
+
if (parsed.data.result.results.length === 0) {
|
|
93
|
+
throw new UsSourceParseError(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, 'The response carried no entries');
|
|
94
|
+
}
|
|
95
|
+
const entries = parsed.data.result.results.map((row) => {
|
|
96
|
+
if (row.id === undefined ||
|
|
97
|
+
row.name === undefined ||
|
|
98
|
+
row.title === undefined ||
|
|
99
|
+
row.metadata_created === undefined) {
|
|
100
|
+
throw new UsSourceParseError(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, 'A catalogue entry arrived without its id, name, title, or creation time');
|
|
101
|
+
}
|
|
102
|
+
const createdAtEpochSeconds = parseCkanTimestamp(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, row.metadata_created, 'metadata_created');
|
|
103
|
+
const modifiedAtEpochSeconds = row.metadata_modified === undefined
|
|
104
|
+
? createdAtEpochSeconds
|
|
105
|
+
: parseCkanTimestamp(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, row.metadata_modified, 'metadata_modified');
|
|
106
|
+
return {
|
|
107
|
+
id: row.id,
|
|
108
|
+
name: row.name,
|
|
109
|
+
title: row.title,
|
|
110
|
+
assetType: row.type ?? '',
|
|
111
|
+
category: row.groups?.[0]?.title ?? null,
|
|
112
|
+
createdAtEpochSeconds,
|
|
113
|
+
viewLastModifiedEpochSeconds: modifiedAtEpochSeconds,
|
|
114
|
+
resourceCount: row.num_resources ?? row.resources?.length ?? 0,
|
|
115
|
+
licenceUrl: (row.license_url ?? row.license_title ?? '').trim(),
|
|
116
|
+
attribution: (row.organization?.title ?? '').trim(),
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
return {
|
|
120
|
+
entries,
|
|
121
|
+
entryCount: entries.length,
|
|
122
|
+
catalogueCount: parsed.data.result.count,
|
|
123
|
+
datasetCount: entries.filter((entry) => entry.assetType === 'dataset').length,
|
|
124
|
+
newestEntry: newestCaliforniaOpenDataEntry(entries),
|
|
125
|
+
categories: countCaliforniaOpenDataCategories(entries),
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Reads one page of the California Open Data catalogue through CKAN.
|
|
130
|
+
*
|
|
131
|
+
* One keyless request covers the page. The Socrata discovery endpoint on the
|
|
132
|
+
* same host answers HTTP 404, while CKAN package search answers JSON.
|
|
133
|
+
*
|
|
134
|
+
* @param options - an optional entry cap and fetch stub
|
|
135
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
136
|
+
*/
|
|
137
|
+
export async function fetchCaliforniaOpenDataCatalogue(options) {
|
|
138
|
+
const fetchImpl = options?.fetchImpl ?? globalThis.fetch;
|
|
139
|
+
const url = buildCaliforniaOpenDataCatalogueUrl(options?.limit === undefined ? {} : { limit: options.limit });
|
|
140
|
+
const response = await httpGet(CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID, url, { fetchImpl });
|
|
141
|
+
return parseCaliforniaOpenDataCataloguePayload(await response.json());
|
|
142
|
+
}
|
|
143
|
+
/** California Open Data catalogue of datasets and resources, keyless through CKAN. */
|
|
144
|
+
export const californiaOpenDataCatalogueAdapter = {
|
|
145
|
+
id: CALIFORNIA_OPEN_DATA_CATALOGUE_SOURCE_ID,
|
|
146
|
+
name: 'California Open Data catalogue',
|
|
147
|
+
auth: 'none',
|
|
148
|
+
description: 'The California Open Data CKAN catalogue, one row per published dataset package, with its title, groups, resources, licence, and publishing organisation.',
|
|
149
|
+
fetchLive: async (options) => fetchCaliforniaOpenDataCatalogue(options?.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }),
|
|
150
|
+
parse: (payload) => parseCaliforniaOpenDataCataloguePayload(payload),
|
|
151
|
+
loadFixture: () => parseCaliforniaOpenDataCataloguePayload(readFixtureJson(CALIFORNIA_OPEN_DATA_FIXTURE_FILENAME)),
|
|
152
|
+
};
|
|
153
|
+
//# sourceMappingURL=californiaOpenDataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"californiaOpenDataCatalogue.js","sourceRoot":"","sources":["../src/californiaOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIpC,gEAAgE;AAChE,MAAM,CAAC,MAAM,wCAAwC,GAAG,gCAAgC,CAAC;AAEzF,8EAA8E;AAC9E,MAAM,CAAC,MAAM,6BAA6B,GAAG,iDAAiD,CAAC;AAE/F,oDAAoD;AACpD,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAC;AA4CvD,oFAAoF;AACpF,MAAM,qCAAqC,GAAG,gDAAgD,CAAC;AAE/F,qFAAqF;AACrF,MAAM,kCAAkC,GAAG,eAAe,CAAC;AAE3D,MAAM,mCAAmC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC5B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,iBAAiB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACxC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC7C,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC/C,YAAY,EAAE,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC9E,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE;IACtE,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,EAAE;CAC3C,CAAC,CAAC;AAEH,MAAM,mCAAmC,GAAG,CAAC,CAAC,MAAM,CAAC;IACnD,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;IACxB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC;QACf,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;QACjB,OAAO,EAAE,CAAC,CAAC,KAAK,CAAC,mCAAmC,CAAC;KACtD,CAAC;CACH,CAAC,CAAC;AAEH,gEAAgE;AAChE,SAAS,kBAAkB,CAAC,QAAgB,EAAE,KAAa,EAAE,KAAa;IACxE,MAAM,UAAU,GAAG,yBAAyB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,CAAC;IAC/E,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACtC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,wCAAwC,KAAK,EAAE,CAAC,CAAC;IAC1F,CAAC;IACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC;AACnC,CAAC;AAED,qEAAqE;AACrE,SAAS,iCAAiC,CACxC,OAA2C;IAE3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,kCAAkC,CAAC;QACtE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,CAC/D,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAC9F;SACA,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,gFAAgF;AAChF,SAAS,6BAA6B,CACpC,OAA2C;IAE3C,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,kBAAkB,CAC1B,wCAAwC,EACxC,yBAAyB,CAC1B,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAC1F,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mCAAmC,CAAC,OAA4B;IAC9E,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,IAAI,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,oCAAoC,CAAC;KACrE,CAAC,CAAC;IACH,OAAO,GAAG,6BAA6B,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,uCAAuC,CACrD,OAAgB;IAEhB,MAAM,MAAM,GAAG,mCAAmC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACtE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,wCAAwC,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/F,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,kBAAkB,CAC1B,wCAAwC,EACxC,iCAAiC,CAClC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAuC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACzF,IACE,GAAG,CAAC,EAAE,KAAK,SAAS;YACpB,GAAG,CAAC,IAAI,KAAK,SAAS;YACtB,GAAG,CAAC,KAAK,KAAK,SAAS;YACvB,GAAG,CAAC,gBAAgB,KAAK,SAAS,EAClC,CAAC;YACD,MAAM,IAAI,kBAAkB,CAC1B,wCAAwC,EACxC,yEAAyE,CAC1E,CAAC;QACJ,CAAC;QACD,MAAM,qBAAqB,GAAG,kBAAkB,CAC9C,wCAAwC,EACxC,GAAG,CAAC,gBAAgB,EACpB,kBAAkB,CACnB,CAAC;QACF,MAAM,sBAAsB,GAC1B,GAAG,CAAC,iBAAiB,KAAK,SAAS;YACjC,CAAC,CAAC,qBAAqB;YACvB,CAAC,CAAC,kBAAkB,CAChB,wCAAwC,EACxC,GAAG,CAAC,iBAAiB,EACrB,mBAAmB,CACpB,CAAC;QACR,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,SAAS,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE;YACzB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,IAAI;YACxC,qBAAqB;YACrB,4BAA4B,EAAE,sBAAsB;YACpD,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,GAAG,CAAC,SAAS,EAAE,MAAM,IAAI,CAAC;YAC9D,UAAU,EAAE,CAAC,GAAG,CAAC,WAAW,IAAI,GAAG,CAAC,aAAa,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;YAC/D,WAAW,EAAE,CAAC,GAAG,CAAC,YAAY,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE;SACpD,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,cAAc,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK;QACxC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7E,WAAW,EAAE,6BAA6B,CAAC,OAAO,CAAC;QACnD,UAAU,EAAE,iCAAiC,CAAC,OAAO,CAAC;KACvD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gCAAgC,CAAC,OAGtD;IACC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACzD,MAAM,GAAG,GAAG,mCAAmC,CAC7C,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAC7D,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,wCAAwC,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAC7F,OAAO,uCAAuC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,sFAAsF;AACtF,MAAM,CAAC,MAAM,kCAAkC,GAA+C;IAC5F,EAAE,EAAE,wCAAwC;IAC5C,IAAI,EAAE,gCAAgC;IACtC,IAAI,EAAE,MAAM;IACZ,WAAW,EACT,0JAA0J;IAC5J,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC3B,gCAAgC,CAC9B,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACzE;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,uCAAuC,CAAC,OAAO,CAAC;IACpE,WAAW,EAAE,GAAG,EAAE,CAChB,uCAAuC,CAAC,eAAe,CAAC,qCAAqC,CAAC,CAAC;CAClG,CAAC"}
|
|
@@ -21,6 +21,10 @@ export interface CdcSocrataCatalogueEntry {
|
|
|
21
21
|
viewCount: number;
|
|
22
22
|
/** How many times the entry has been downloaded. */
|
|
23
23
|
downloadCount: number;
|
|
24
|
+
/** Licence URL the portal attaches to the entry, or "" when it supplies none. */
|
|
25
|
+
licenceUrl: string;
|
|
26
|
+
/** Publishing organisation the portal attaches to the entry, or "" when it supplies none. */
|
|
27
|
+
attribution: string;
|
|
24
28
|
}
|
|
25
29
|
/** One category and how many catalogue entries carry it. */
|
|
26
30
|
export interface CdcSocrataCategoryCount {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cdcSocrataCatalogue.d.ts","sourceRoot":"","sources":["../src/cdcSocrataCatalogue.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"cdcSocrataCatalogue.d.ts","sourceRoot":"","sources":["../src/cdcSocrataCatalogue.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,uDAAuD;AACvD,eAAO,MAAM,+BAA+B,0BAA0B,CAAC;AAEvE,0DAA0D;AAC1D,MAAM,WAAW,wBAAwB;IACvC,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,kEAAkE;IAClE,IAAI,EAAE,MAAM,CAAC;IACb,yEAAyE;IACzE,SAAS,EAAE,MAAM,CAAC;IAClB,qFAAqF;IACrF,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,qEAAqE;IACrE,QAAQ,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,yEAAyE;IACzE,4BAA4B,EAAE,MAAM,CAAC;IACrC,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,iFAAiF;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,4DAA4D;AAC5D,MAAM,WAAW,uBAAuB;IACtC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,qDAAqD;AACrD,MAAM,WAAW,mBAAmB;IAClC,gDAAgD;IAChD,OAAO,EAAE,wBAAwB,EAAE,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,WAAW,EAAE,wBAAwB,CAAC;IACtC;;;OAGG;IACH,UAAU,EAAE,uBAAuB,EAAE,CAAC;CACvC;AAED,8CAA8C;AAC9C,eAAO,MAAM,8BAA8B,wCAAwC,CAAC;AAEpF,oDAAoD;AACpD,eAAO,MAAM,2BAA2B,KAAK,CAAC;AAmD9C;;;;;;;;GAQG;AACH,wBAAgB,2BAA2B,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAKhF;AAED;;;;;;;;;GASG;AACH,wBAAgB,+BAA+B,CAAC,OAAO,EAAE,OAAO,GAAG,mBAAmB,CAwCrF;AAED;;;;;;;;GAQG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,CAAC,EAAE;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAO/B;AAED,uEAAuE;AACvE,eAAO,MAAM,0BAA0B,EAAE,aAAa,CAAC,mBAAmB,CAazE,CAAC"}
|
|
@@ -2,6 +2,7 @@ import { z } from 'zod';
|
|
|
2
2
|
import { UsSourceParseError } from './errors.js';
|
|
3
3
|
import { httpGet } from './http.js';
|
|
4
4
|
import { readFixtureJson } from './fixtures.js';
|
|
5
|
+
import { parseSocrataCatalogueMetadata } from './socrataCatalogueMetadata.js';
|
|
5
6
|
/** Adapter id for the cdc-socrata-catalogue source. */
|
|
6
7
|
export const CDC_SOCRATA_CATALOGUE_SOURCE_ID = 'cdc-socrata-catalogue';
|
|
7
8
|
/** Base URL for the CDC Socrata catalogue. */
|
|
@@ -24,6 +25,7 @@ const CDC_SOCRATA_CATALOGUE_ENTRY_SCHEMA = z.object({
|
|
|
24
25
|
viewLastModified: z.number().optional(),
|
|
25
26
|
viewCount: z.number().optional(),
|
|
26
27
|
downloadCount: z.number().optional(),
|
|
28
|
+
metadata: z.unknown().optional(),
|
|
27
29
|
});
|
|
28
30
|
const CDC_SOCRATA_CATALOGUE_PAYLOAD_SCHEMA = z.array(CDC_SOCRATA_CATALOGUE_ENTRY_SCHEMA);
|
|
29
31
|
/** Counts entries per category, most entries first, ties by name. */
|
|
@@ -92,6 +94,7 @@ export function parseCdcSocrataCataloguePayload(payload) {
|
|
|
92
94
|
viewLastModifiedEpochSeconds: row.viewLastModified ?? row.createdAt,
|
|
93
95
|
viewCount: row.viewCount ?? 0,
|
|
94
96
|
downloadCount: row.downloadCount ?? 0,
|
|
97
|
+
...parseSocrataCatalogueMetadata(row.metadata),
|
|
95
98
|
};
|
|
96
99
|
});
|
|
97
100
|
return {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"cdcSocrataCatalogue.js","sourceRoot":"","sources":["../src/cdcSocrataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"cdcSocrataCatalogue.js","sourceRoot":"","sources":["../src/cdcSocrataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAC;AAG9E,uDAAuD;AACvD,MAAM,CAAC,MAAM,+BAA+B,GAAG,uBAAuB,CAAC;AAkDvE,8CAA8C;AAC9C,MAAM,CAAC,MAAM,8BAA8B,GAAG,qCAAqC,CAAC;AAEpF,oDAAoD;AACpD,MAAM,CAAC,MAAM,2BAA2B,GAAG,EAAE,CAAC;AAE9C,6EAA6E;AAC7E,MAAM,sCAAsC,GAAG,uCAAuC,CAAC;AAEvF,qFAAqF;AACrF,MAAM,yBAAyB,GAAG,eAAe,CAAC;AAElD,4EAA4E;AAC5E,8DAA8D;AAC9D,MAAM,kCAAkC,GAAG,CAAC,CAAC,MAAM,CAAC;IAClD,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAEH,MAAM,oCAAoC,GAAG,CAAC,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;AAEzF,qEAAqE;AACrE,SAAS,yBAAyB,CAAC,OAAmC;IACpE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,yBAAyB,CAAC;QAC7D,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,CAC/D,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAC9F;SACA,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,gFAAgF;AAChF,SAAS,qBAAqB,CAAC,OAAmC;IAChE,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,kBAAkB,CAAC,+BAA+B,EAAE,yBAAyB,CAAC,CAAC;IAC3F,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAC1F,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,2BAA2B,CAAC,OAA4B;IACtE,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,2BAA2B,CAAC;KAC7D,CAAC,CAAC;IACH,OAAO,uCAAuC,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AACpE,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,+BAA+B,CAAC,OAAgB;IAC9D,MAAM,MAAM,GAAG,oCAAoC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACvE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,+BAA+B,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACtF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,kBAAkB,CAC1B,+BAA+B,EAC/B,iCAAiC,CAClC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAA+B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAClE,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAClF,MAAM,IAAI,kBAAkB,CAC1B,+BAA+B,EAC/B,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,4BAA4B,EAAE,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS;YACnE,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;YAC7B,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC;YACrC,GAAG,6BAA6B,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7E,WAAW,EAAE,qBAAqB,CAAC,OAAO,CAAC;QAC3C,UAAU,EAAE,yBAAyB,CAAC,OAAO,CAAC;KAC/C,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,wBAAwB,CAAC,OAG9C;IACC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACzD,MAAM,GAAG,GAAG,2BAA2B,CACrC,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAC7D,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,+BAA+B,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACpF,OAAO,+BAA+B,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AAChE,CAAC;AAED,uEAAuE;AACvE,MAAM,CAAC,MAAM,0BAA0B,GAAuC;IAC5E,EAAE,EAAE,+BAA+B;IACnC,IAAI,EAAE,uBAAuB;IAC7B,IAAI,EAAE,MAAM;IACZ,WAAW,EACT,iIAAiI;IACnI,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC3B,wBAAwB,CACtB,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACzE;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,+BAA+B,CAAC,OAAO,CAAC;IAC5D,WAAW,EAAE,GAAG,EAAE,CAChB,+BAA+B,CAAC,eAAe,CAAC,sCAAsC,CAAC,CAAC;CAC3F,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { UsDataAdapter } from './types.js';
|
|
2
|
+
/** Adapter id for the Chicago Data Portal catalogue source. */
|
|
3
|
+
export declare const CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID = "chicago-open-data-catalogue";
|
|
4
|
+
/** Base URL for the Chicago Data Portal Socrata catalogue. */
|
|
5
|
+
export declare const CHICAGO_OPEN_DATA_API_BASE = "https://data.cityofchicago.org/api/views.json";
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export declare const CHICAGO_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** One catalogue entry from the Chicago Socrata discovery API. */
|
|
9
|
+
export interface ChicagoOpenDataCatalogueEntry {
|
|
10
|
+
/** Four-by-four Socrata dataset id, e.g. "p4pq-bxn9". */
|
|
11
|
+
id: string;
|
|
12
|
+
/** Human-readable title of the dataset, chart, story, map, or file. */
|
|
13
|
+
name: string;
|
|
14
|
+
/** What the entry is: "dataset", "chart", "story", "map", or "file". */
|
|
15
|
+
assetType: string;
|
|
16
|
+
/** City topic the entry belongs to, or null when the catalogue lists none. */
|
|
17
|
+
category: string | null;
|
|
18
|
+
/** How the entry is rendered, e.g. "tabular" or "story". */
|
|
19
|
+
viewType: string;
|
|
20
|
+
/** When the entry was created, in seconds since the Unix epoch. */
|
|
21
|
+
createdAtEpochSeconds: number;
|
|
22
|
+
/** When the entry was last modified, in seconds since the Unix epoch. */
|
|
23
|
+
viewLastModifiedEpochSeconds: number;
|
|
24
|
+
/** How many times the entry has been viewed. */
|
|
25
|
+
viewCount: number;
|
|
26
|
+
/** How many times the entry has been downloaded. */
|
|
27
|
+
downloadCount: number;
|
|
28
|
+
/** Licence text or URL the portal attaches, or "" when it supplies none. */
|
|
29
|
+
licenceUrl: string;
|
|
30
|
+
/** Publishing organisation the portal attaches, or "" when it supplies none. */
|
|
31
|
+
attribution: string;
|
|
32
|
+
}
|
|
33
|
+
/** One city topic and how many catalogue entries carry it. */
|
|
34
|
+
export interface ChicagoOpenDataCategoryCount {
|
|
35
|
+
category: string;
|
|
36
|
+
entryCount: number;
|
|
37
|
+
}
|
|
38
|
+
/** One page of the Chicago catalogue, folded for a story. */
|
|
39
|
+
export interface ChicagoOpenDataCatalogue {
|
|
40
|
+
/** One entry per catalogue item in the page. */
|
|
41
|
+
entries: ChicagoOpenDataCatalogueEntry[];
|
|
42
|
+
entryCount: number;
|
|
43
|
+
/** Entries whose asset type is "dataset". */
|
|
44
|
+
datasetCount: number;
|
|
45
|
+
/** Entry with the latest creation time. Ties go to the earlier list entry. */
|
|
46
|
+
newestEntry: ChicagoOpenDataCatalogueEntry;
|
|
47
|
+
/**
|
|
48
|
+
* Categories in the page, most entries first, ties broken by name. An entry
|
|
49
|
+
* the catalogue lists without a category is counted under "Uncategorised".
|
|
50
|
+
*/
|
|
51
|
+
categories: ChicagoOpenDataCategoryCount[];
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Builds the request URL for one page of the Chicago catalogue.
|
|
55
|
+
*
|
|
56
|
+
* The endpoint is keyless. The service answers the whole catalogue by default,
|
|
57
|
+
* so the adapter asks for a short page.
|
|
58
|
+
*
|
|
59
|
+
* @param options - an optional entry cap
|
|
60
|
+
* @returns the full request URL
|
|
61
|
+
*/
|
|
62
|
+
export declare function buildChicagoOpenDataCatalogueUrl(options?: {
|
|
63
|
+
limit?: number;
|
|
64
|
+
}): string;
|
|
65
|
+
/**
|
|
66
|
+
* Parses one page of the Chicago catalogue into the entries and topic counts.
|
|
67
|
+
*
|
|
68
|
+
* An entry without an id, a name, or a creation time is a changed shape and
|
|
69
|
+
* stops the parse. An empty page also stops the parse, because a catalogue
|
|
70
|
+
* with no entries is not a usable answer.
|
|
71
|
+
*
|
|
72
|
+
* @param payload - the JSON body from the Chicago Socrata discovery API
|
|
73
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
74
|
+
*/
|
|
75
|
+
export declare function parseChicagoOpenDataCataloguePayload(payload: unknown): ChicagoOpenDataCatalogue;
|
|
76
|
+
/**
|
|
77
|
+
* Reads one page of the Chicago Data Portal catalogue.
|
|
78
|
+
*
|
|
79
|
+
* One keyless request covers the page and reports what the city portal holds,
|
|
80
|
+
* without reading any dataset's rows.
|
|
81
|
+
*
|
|
82
|
+
* @param options - an optional entry cap and fetch stub
|
|
83
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
84
|
+
*/
|
|
85
|
+
export declare function fetchChicagoOpenDataCatalogue(options?: {
|
|
86
|
+
limit?: number;
|
|
87
|
+
fetchImpl?: typeof globalThis.fetch;
|
|
88
|
+
}): Promise<ChicagoOpenDataCatalogue>;
|
|
89
|
+
/** Chicago Data Portal catalogue of datasets, charts, maps, and stories, keyless. */
|
|
90
|
+
export declare const chicagoOpenDataCatalogueAdapter: UsDataAdapter<ChicagoOpenDataCatalogue>;
|
|
91
|
+
//# sourceMappingURL=chicagoOpenDataCatalogue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chicagoOpenDataCatalogue.d.ts","sourceRoot":"","sources":["../src/chicagoOpenDataCatalogue.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,+DAA+D;AAC/D,eAAO,MAAM,qCAAqC,gCAAgC,CAAC;AAEnF,8DAA8D;AAC9D,eAAO,MAAM,0BAA0B,kDAAkD,CAAC;AAE1F,oDAAoD;AACpD,eAAO,MAAM,iCAAiC,KAAK,CAAC;AAEpD,kEAAkE;AAClE,MAAM,WAAW,6BAA6B;IAC5C,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,IAAI,EAAE,MAAM,CAAC;IACb,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB,8EAA8E;IAC9E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,yEAAyE;IACzE,4BAA4B,EAAE,MAAM,CAAC;IACrC,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,8DAA8D;AAC9D,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,6DAA6D;AAC7D,MAAM,WAAW,wBAAwB;IACvC,gDAAgD;IAChD,OAAO,EAAE,6BAA6B,EAAE,CAAC;IACzC,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,WAAW,EAAE,6BAA6B,CAAC;IAC3C;;;OAGG;IACH,UAAU,EAAE,4BAA4B,EAAE,CAAC;CAC5C;AAuDD;;;;;;;;GAQG;AACH,wBAAgB,gCAAgC,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAKrF;AAED;;;;;;;;;GASG;AACH,wBAAgB,oCAAoC,CAAC,OAAO,EAAE,OAAO,GAAG,wBAAwB,CAwC/F;AAED;;;;;;;;GAQG;AACH,wBAAsB,6BAA6B,CAAC,OAAO,CAAC,EAAE;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAOpC;AAED,qFAAqF;AACrF,eAAO,MAAM,+BAA+B,EAAE,aAAa,CAAC,wBAAwB,CAanF,CAAC"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { UsSourceParseError } from './errors.js';
|
|
3
|
+
import { readFixtureJson } from './fixtures.js';
|
|
4
|
+
import { httpGet } from './http.js';
|
|
5
|
+
import { parseSocrataCatalogueMetadata } from './socrataCatalogueMetadata.js';
|
|
6
|
+
/** Adapter id for the Chicago Data Portal catalogue source. */
|
|
7
|
+
export const CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID = 'chicago-open-data-catalogue';
|
|
8
|
+
/** Base URL for the Chicago Data Portal Socrata catalogue. */
|
|
9
|
+
export const CHICAGO_OPEN_DATA_API_BASE = 'https://data.cityofchicago.org/api/views.json';
|
|
10
|
+
/** How many catalogue entries one request reads. */
|
|
11
|
+
export const CHICAGO_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
12
|
+
/** Committed snapshot, so a build works when the Chicago host is unreachable. */
|
|
13
|
+
const CHICAGO_OPEN_DATA_FIXTURE_FILENAME = 'chicago-open-data-catalogue-2026-11-08.json';
|
|
14
|
+
/** Category bucket the adapter uses for an entry the catalogue lists without one. */
|
|
15
|
+
const CHICAGO_OPEN_DATA_UNCATEGORISED = 'Uncategorised';
|
|
16
|
+
// The discovery API sends a bare JSON array. A value the catalogue does not
|
|
17
|
+
// carry arrives as null, and every count arrives as a number.
|
|
18
|
+
const CHICAGO_OPEN_DATA_ENTRY_SCHEMA = z.object({
|
|
19
|
+
id: z.string().optional(),
|
|
20
|
+
name: z.string().optional(),
|
|
21
|
+
assetType: z.string().optional(),
|
|
22
|
+
category: z.string().nullable().optional(),
|
|
23
|
+
viewType: z.string().optional(),
|
|
24
|
+
createdAt: z.number().optional(),
|
|
25
|
+
viewLastModified: z.number().optional(),
|
|
26
|
+
viewCount: z.number().optional(),
|
|
27
|
+
downloadCount: z.number().optional(),
|
|
28
|
+
metadata: z.unknown().optional(),
|
|
29
|
+
});
|
|
30
|
+
const CHICAGO_OPEN_DATA_PAYLOAD_SCHEMA = z.array(CHICAGO_OPEN_DATA_ENTRY_SCHEMA);
|
|
31
|
+
/** Counts entries per category, most entries first, ties by name. */
|
|
32
|
+
function countChicagoOpenDataCategories(entries) {
|
|
33
|
+
const counts = new Map();
|
|
34
|
+
for (const entry of entries) {
|
|
35
|
+
const category = entry.category ?? CHICAGO_OPEN_DATA_UNCATEGORISED;
|
|
36
|
+
counts.set(category, (counts.get(category) ?? 0) + 1);
|
|
37
|
+
}
|
|
38
|
+
return [...counts.entries()]
|
|
39
|
+
.sort(([leftCategory, leftCount], [rightCategory, rightCount]) => leftCount === rightCount ? leftCategory.localeCompare(rightCategory) : rightCount - leftCount)
|
|
40
|
+
.map(([category, entryCount]) => ({ category, entryCount }));
|
|
41
|
+
}
|
|
42
|
+
/** Picks the entry with the latest creation time, ties to the earlier entry. */
|
|
43
|
+
function newestChicagoOpenDataEntry(entries) {
|
|
44
|
+
const first = entries[0];
|
|
45
|
+
if (first === undefined) {
|
|
46
|
+
throw new UsSourceParseError(CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID, 'No entries to summarise');
|
|
47
|
+
}
|
|
48
|
+
return entries.reduce((best, entry) => (entry.createdAtEpochSeconds > best.createdAtEpochSeconds ? entry : best), first);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Builds the request URL for one page of the Chicago catalogue.
|
|
52
|
+
*
|
|
53
|
+
* The endpoint is keyless. The service answers the whole catalogue by default,
|
|
54
|
+
* so the adapter asks for a short page.
|
|
55
|
+
*
|
|
56
|
+
* @param options - an optional entry cap
|
|
57
|
+
* @returns the full request URL
|
|
58
|
+
*/
|
|
59
|
+
export function buildChicagoOpenDataCatalogueUrl(options) {
|
|
60
|
+
const params = new URLSearchParams({
|
|
61
|
+
limit: String(options?.limit ?? CHICAGO_OPEN_DATA_CATALOGUE_LIMIT),
|
|
62
|
+
});
|
|
63
|
+
return `${CHICAGO_OPEN_DATA_API_BASE}?${params.toString()}`;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Parses one page of the Chicago catalogue into the entries and topic counts.
|
|
67
|
+
*
|
|
68
|
+
* An entry without an id, a name, or a creation time is a changed shape and
|
|
69
|
+
* stops the parse. An empty page also stops the parse, because a catalogue
|
|
70
|
+
* with no entries is not a usable answer.
|
|
71
|
+
*
|
|
72
|
+
* @param payload - the JSON body from the Chicago Socrata discovery API
|
|
73
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
74
|
+
*/
|
|
75
|
+
export function parseChicagoOpenDataCataloguePayload(payload) {
|
|
76
|
+
const parsed = CHICAGO_OPEN_DATA_PAYLOAD_SCHEMA.safeParse(payload);
|
|
77
|
+
if (!parsed.success) {
|
|
78
|
+
throw new UsSourceParseError(CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID, parsed.error.message);
|
|
79
|
+
}
|
|
80
|
+
if (parsed.data.length === 0) {
|
|
81
|
+
throw new UsSourceParseError(CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID, 'The response carried no entries');
|
|
82
|
+
}
|
|
83
|
+
const entries = parsed.data.map((row) => {
|
|
84
|
+
if (row.id === undefined || row.name === undefined || row.createdAt === undefined) {
|
|
85
|
+
throw new UsSourceParseError(CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID, 'A catalogue entry arrived without its id, name, or creation time');
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
id: row.id,
|
|
89
|
+
name: row.name,
|
|
90
|
+
assetType: row.assetType ?? '',
|
|
91
|
+
category: row.category ?? null,
|
|
92
|
+
viewType: row.viewType ?? '',
|
|
93
|
+
createdAtEpochSeconds: row.createdAt,
|
|
94
|
+
viewLastModifiedEpochSeconds: row.viewLastModified ?? row.createdAt,
|
|
95
|
+
viewCount: row.viewCount ?? 0,
|
|
96
|
+
downloadCount: row.downloadCount ?? 0,
|
|
97
|
+
...parseSocrataCatalogueMetadata(row.metadata),
|
|
98
|
+
};
|
|
99
|
+
});
|
|
100
|
+
return {
|
|
101
|
+
entries,
|
|
102
|
+
entryCount: entries.length,
|
|
103
|
+
datasetCount: entries.filter((entry) => entry.assetType === 'dataset').length,
|
|
104
|
+
newestEntry: newestChicagoOpenDataEntry(entries),
|
|
105
|
+
categories: countChicagoOpenDataCategories(entries),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Reads one page of the Chicago Data Portal catalogue.
|
|
110
|
+
*
|
|
111
|
+
* One keyless request covers the page and reports what the city portal holds,
|
|
112
|
+
* without reading any dataset's rows.
|
|
113
|
+
*
|
|
114
|
+
* @param options - an optional entry cap and fetch stub
|
|
115
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
116
|
+
*/
|
|
117
|
+
export async function fetchChicagoOpenDataCatalogue(options) {
|
|
118
|
+
const fetchImpl = options?.fetchImpl ?? globalThis.fetch;
|
|
119
|
+
const url = buildChicagoOpenDataCatalogueUrl(options?.limit === undefined ? {} : { limit: options.limit });
|
|
120
|
+
const response = await httpGet(CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID, url, { fetchImpl });
|
|
121
|
+
return parseChicagoOpenDataCataloguePayload(await response.json());
|
|
122
|
+
}
|
|
123
|
+
/** Chicago Data Portal catalogue of datasets, charts, maps, and stories, keyless. */
|
|
124
|
+
export const chicagoOpenDataCatalogueAdapter = {
|
|
125
|
+
id: CHICAGO_OPEN_DATA_CATALOGUE_SOURCE_ID,
|
|
126
|
+
name: 'Chicago Data Portal catalogue',
|
|
127
|
+
auth: 'none',
|
|
128
|
+
description: 'The Chicago Data Portal Socrata catalogue, one row per published dataset, chart, map, or story, with its city topic, view count, and download count.',
|
|
129
|
+
fetchLive: async (options) => fetchChicagoOpenDataCatalogue(options?.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }),
|
|
130
|
+
parse: (payload) => parseChicagoOpenDataCataloguePayload(payload),
|
|
131
|
+
loadFixture: () => parseChicagoOpenDataCataloguePayload(readFixtureJson(CHICAGO_OPEN_DATA_FIXTURE_FILENAME)),
|
|
132
|
+
};
|
|
133
|
+
//# sourceMappingURL=chicagoOpenDataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chicagoOpenDataCatalogue.js","sourceRoot":"","sources":["../src/chicagoOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAC;AAG9E,+DAA+D;AAC/D,MAAM,CAAC,MAAM,qCAAqC,GAAG,6BAA6B,CAAC;AAEnF,8DAA8D;AAC9D,MAAM,CAAC,MAAM,0BAA0B,GAAG,+CAA+C,CAAC;AAE1F,oDAAoD;AACpD,MAAM,CAAC,MAAM,iCAAiC,GAAG,EAAE,CAAC;AAkDpD,iFAAiF;AACjF,MAAM,kCAAkC,GAAG,6CAA6C,CAAC;AAEzF,qFAAqF;AACrF,MAAM,+BAA+B,GAAG,eAAe,CAAC;AAExD,4EAA4E;AAC5E,8DAA8D;AAC9D,MAAM,8BAA8B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAEH,MAAM,gCAAgC,GAAG,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;AAEjF,qEAAqE;AACrE,SAAS,8BAA8B,CACrC,OAAwC;IAExC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,+BAA+B,CAAC;QACnE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,CAC/D,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAC9F;SACA,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,gFAAgF;AAChF,SAAS,0BAA0B,CACjC,OAAwC;IAExC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,kBAAkB,CAAC,qCAAqC,EAAE,yBAAyB,CAAC,CAAC;IACjG,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAC1F,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,gCAAgC,CAAC,OAA4B;IAC3E,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,iCAAiC,CAAC;KACnE,CAAC,CAAC;IACH,OAAO,GAAG,0BAA0B,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC9D,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,oCAAoC,CAAC,OAAgB;IACnE,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACnE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,qCAAqC,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC5F,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,kBAAkB,CAC1B,qCAAqC,EACrC,iCAAiC,CAClC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAoC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACvE,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAClF,MAAM,IAAI,kBAAkB,CAC1B,qCAAqC,EACrC,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,4BAA4B,EAAE,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS;YACnE,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;YAC7B,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC;YACrC,GAAG,6BAA6B,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7E,WAAW,EAAE,0BAA0B,CAAC,OAAO,CAAC;QAChD,UAAU,EAAE,8BAA8B,CAAC,OAAO,CAAC;KACpD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,OAGnD;IACC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACzD,MAAM,GAAG,GAAG,gCAAgC,CAC1C,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAC7D,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,qCAAqC,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IAC1F,OAAO,oCAAoC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AACrE,CAAC;AAED,qFAAqF;AACrF,MAAM,CAAC,MAAM,+BAA+B,GAA4C;IACtF,EAAE,EAAE,qCAAqC;IACzC,IAAI,EAAE,+BAA+B;IACrC,IAAI,EAAE,MAAM;IACZ,WAAW,EACT,sJAAsJ;IACxJ,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC3B,6BAA6B,CAC3B,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACzE;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,oCAAoC,CAAC,OAAO,CAAC;IACjE,WAAW,EAAE,GAAG,EAAE,CAChB,oCAAoC,CAAC,eAAe,CAAC,kCAAkC,CAAC,CAAC;CAC5F,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type SocrataCatalogue, type SocrataCatalogueCategoryCount, type SocrataCatalogueEntry } from './socrataCatalogue.js';
|
|
2
|
+
/** Adapter id for the Colorado Open Data catalogue source. */
|
|
3
|
+
export declare const COLORADO_OPEN_DATA_CATALOGUE_SOURCE_ID = "colorado-open-data-catalogue";
|
|
4
|
+
/** Base URL for the Colorado Open Data Socrata catalogue. */
|
|
5
|
+
export declare const COLORADO_OPEN_DATA_API_BASE = "https://data.colorado.gov/api/views.json";
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export declare const COLORADO_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** One catalogue entry from the Colorado Socrata discovery API. */
|
|
9
|
+
export type ColoradoOpenDataCatalogueEntry = SocrataCatalogueEntry;
|
|
10
|
+
/** One state topic and how many catalogue entries carry it. */
|
|
11
|
+
export type ColoradoOpenDataCategoryCount = SocrataCatalogueCategoryCount;
|
|
12
|
+
/** One page of the Colorado catalogue, folded for a story. */
|
|
13
|
+
export type ColoradoOpenDataCatalogue = SocrataCatalogue;
|
|
14
|
+
/**
|
|
15
|
+
* Builds the request URL for one page of the Colorado catalogue.
|
|
16
|
+
*
|
|
17
|
+
* @param options - an optional entry cap
|
|
18
|
+
* @returns the full request URL
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildColoradoOpenDataCatalogueUrl(options?: {
|
|
21
|
+
limit?: number;
|
|
22
|
+
}): string;
|
|
23
|
+
/**
|
|
24
|
+
* Parses one page of the Colorado catalogue into entries and topic counts.
|
|
25
|
+
*
|
|
26
|
+
* @param payload - the JSON body from the Colorado Socrata discovery API
|
|
27
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
28
|
+
*/
|
|
29
|
+
export declare function parseColoradoOpenDataCataloguePayload(payload: unknown): ColoradoOpenDataCatalogue;
|
|
30
|
+
/**
|
|
31
|
+
* Reads one page of the Colorado Open Data catalogue.
|
|
32
|
+
*
|
|
33
|
+
* @param options - an optional entry cap and fetch stub
|
|
34
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
35
|
+
*/
|
|
36
|
+
export declare function fetchColoradoOpenDataCatalogue(options?: {
|
|
37
|
+
limit?: number;
|
|
38
|
+
fetchImpl?: typeof globalThis.fetch;
|
|
39
|
+
}): Promise<ColoradoOpenDataCatalogue>;
|
|
40
|
+
/** Colorado Open Data catalogue of datasets, maps, charts, and linked resources, keyless. */
|
|
41
|
+
export declare const coloradoOpenDataCatalogueAdapter: import("./types.js").UsDataAdapter<SocrataCatalogue>;
|
|
42
|
+
//# sourceMappingURL=coloradoOpenDataCatalogue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coloradoOpenDataCatalogue.d.ts","sourceRoot":"","sources":["../src/coloradoOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,qBAAqB,EAC3B,MAAM,uBAAuB,CAAC;AAE/B,8DAA8D;AAC9D,eAAO,MAAM,sCAAsC,iCAAiC,CAAC;AAErF,6DAA6D;AAC7D,eAAO,MAAM,2BAA2B,6CAA6C,CAAC;AAEtF,oDAAoD;AACpD,eAAO,MAAM,kCAAkC,KAA0B,CAAC;AAE1E,mEAAmE;AACnE,MAAM,MAAM,8BAA8B,GAAG,qBAAqB,CAAC;AAEnE,+DAA+D;AAC/D,MAAM,MAAM,6BAA6B,GAAG,6BAA6B,CAAC;AAE1E,8DAA8D;AAC9D,MAAM,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;AAKzD;;;;;GAKG;AACH,wBAAgB,iCAAiC,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAEtF;AAED;;;;;GAKG;AACH,wBAAgB,qCAAqC,CAAC,OAAO,EAAE,OAAO,GAAG,yBAAyB,CAEjG;AAED;;;;;GAKG;AACH,wBAAsB,8BAA8B,CAAC,OAAO,CAAC,EAAE;IAC7D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,yBAAyB,CAAC,CAKrC;AAED,6FAA6F;AAC7F,eAAO,MAAM,gCAAgC,sDAO3C,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { buildSocrataCatalogueUrl, createSocrataCatalogueAdapter, fetchSocrataCatalogue, parseSocrataCataloguePayload, SOCRATA_CATALOGUE_LIMIT, } from './socrataCatalogue.js';
|
|
2
|
+
/** Adapter id for the Colorado Open Data catalogue source. */
|
|
3
|
+
export const COLORADO_OPEN_DATA_CATALOGUE_SOURCE_ID = 'colorado-open-data-catalogue';
|
|
4
|
+
/** Base URL for the Colorado Open Data Socrata catalogue. */
|
|
5
|
+
export const COLORADO_OPEN_DATA_API_BASE = 'https://data.colorado.gov/api/views.json';
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export const COLORADO_OPEN_DATA_CATALOGUE_LIMIT = SOCRATA_CATALOGUE_LIMIT;
|
|
8
|
+
/** Committed snapshot, so a build works when the Colorado host is unreachable. */
|
|
9
|
+
const COLORADO_OPEN_DATA_FIXTURE_FILENAME = 'colorado-open-data-catalogue-2026-11-08.json';
|
|
10
|
+
/**
|
|
11
|
+
* Builds the request URL for one page of the Colorado catalogue.
|
|
12
|
+
*
|
|
13
|
+
* @param options - an optional entry cap
|
|
14
|
+
* @returns the full request URL
|
|
15
|
+
*/
|
|
16
|
+
export function buildColoradoOpenDataCatalogueUrl(options) {
|
|
17
|
+
return buildSocrataCatalogueUrl(COLORADO_OPEN_DATA_API_BASE, options);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Parses one page of the Colorado catalogue into entries and topic counts.
|
|
21
|
+
*
|
|
22
|
+
* @param payload - the JSON body from the Colorado Socrata discovery API
|
|
23
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
24
|
+
*/
|
|
25
|
+
export function parseColoradoOpenDataCataloguePayload(payload) {
|
|
26
|
+
return parseSocrataCataloguePayload(COLORADO_OPEN_DATA_CATALOGUE_SOURCE_ID, payload);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Reads one page of the Colorado Open Data catalogue.
|
|
30
|
+
*
|
|
31
|
+
* @param options - an optional entry cap and fetch stub
|
|
32
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
33
|
+
*/
|
|
34
|
+
export async function fetchColoradoOpenDataCatalogue(options) {
|
|
35
|
+
return fetchSocrataCatalogue({ sourceId: COLORADO_OPEN_DATA_CATALOGUE_SOURCE_ID, apiBase: COLORADO_OPEN_DATA_API_BASE }, options);
|
|
36
|
+
}
|
|
37
|
+
/** Colorado Open Data catalogue of datasets, maps, charts, and linked resources, keyless. */
|
|
38
|
+
export const coloradoOpenDataCatalogueAdapter = createSocrataCatalogueAdapter({
|
|
39
|
+
sourceId: COLORADO_OPEN_DATA_CATALOGUE_SOURCE_ID,
|
|
40
|
+
name: 'Colorado Open Data catalogue',
|
|
41
|
+
description: 'The Colorado Open Data Socrata catalogue, one row per published dataset, map, chart, or linked resource, with its state topic, view count, and download count.',
|
|
42
|
+
apiBase: COLORADO_OPEN_DATA_API_BASE,
|
|
43
|
+
fixtureFilename: COLORADO_OPEN_DATA_FIXTURE_FILENAME,
|
|
44
|
+
});
|
|
45
|
+
//# sourceMappingURL=coloradoOpenDataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"coloradoOpenDataCatalogue.js","sourceRoot":"","sources":["../src/coloradoOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,6BAA6B,EAC7B,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GAIxB,MAAM,uBAAuB,CAAC;AAE/B,8DAA8D;AAC9D,MAAM,CAAC,MAAM,sCAAsC,GAAG,8BAA8B,CAAC;AAErF,6DAA6D;AAC7D,MAAM,CAAC,MAAM,2BAA2B,GAAG,0CAA0C,CAAC;AAEtF,oDAAoD;AACpD,MAAM,CAAC,MAAM,kCAAkC,GAAG,uBAAuB,CAAC;AAW1E,kFAAkF;AAClF,MAAM,mCAAmC,GAAG,8CAA8C,CAAC;AAE3F;;;;;GAKG;AACH,MAAM,UAAU,iCAAiC,CAAC,OAA4B;IAC5E,OAAO,wBAAwB,CAAC,2BAA2B,EAAE,OAAO,CAAC,CAAC;AACxE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qCAAqC,CAAC,OAAgB;IACpE,OAAO,4BAA4B,CAAC,sCAAsC,EAAE,OAAO,CAAC,CAAC;AACvF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,8BAA8B,CAAC,OAGpD;IACC,OAAO,qBAAqB,CAC1B,EAAE,QAAQ,EAAE,sCAAsC,EAAE,OAAO,EAAE,2BAA2B,EAAE,EAC1F,OAAO,CACR,CAAC;AACJ,CAAC;AAED,6FAA6F;AAC7F,MAAM,CAAC,MAAM,gCAAgC,GAAG,6BAA6B,CAAC;IAC5E,QAAQ,EAAE,sCAAsC;IAChD,IAAI,EAAE,8BAA8B;IACpC,WAAW,EACT,gKAAgK;IAClK,OAAO,EAAE,2BAA2B;IACpC,eAAe,EAAE,mCAAmC;CACrD,CAAC,CAAC"}
|