@usa-open-data-connectors/usa-sources 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +229 -32
- package/dist/btsTransportationStats.d.ts +4 -0
- package/dist/btsTransportationStats.d.ts.map +1 -1
- package/dist/btsTransportationStats.js +3 -0
- package/dist/btsTransportationStats.js.map +1 -1
- package/dist/californiaOpenDataCatalogue.d.ts +84 -0
- package/dist/californiaOpenDataCatalogue.d.ts.map +1 -0
- package/dist/californiaOpenDataCatalogue.js +153 -0
- package/dist/californiaOpenDataCatalogue.js.map +1 -0
- package/dist/cdcSocrataCatalogue.d.ts +4 -0
- package/dist/cdcSocrataCatalogue.d.ts.map +1 -1
- package/dist/cdcSocrataCatalogue.js +3 -0
- package/dist/cdcSocrataCatalogue.js.map +1 -1
- package/dist/chicagoOpenDataCatalogue.d.ts +91 -0
- package/dist/chicagoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/chicagoOpenDataCatalogue.js +133 -0
- package/dist/chicagoOpenDataCatalogue.js.map +1 -0
- package/dist/coloradoOpenDataCatalogue.d.ts +42 -0
- package/dist/coloradoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/coloradoOpenDataCatalogue.js +45 -0
- package/dist/coloradoOpenDataCatalogue.js.map +1 -0
- package/dist/fccSocrataCatalogue.d.ts +91 -0
- package/dist/fccSocrataCatalogue.d.ts.map +1 -0
- package/dist/fccSocrataCatalogue.js +133 -0
- package/dist/fccSocrataCatalogue.js.map +1 -0
- package/dist/federalRegisterDocuments.d.ts +78 -0
- package/dist/federalRegisterDocuments.d.ts.map +1 -0
- package/dist/federalRegisterDocuments.js +119 -0
- package/dist/federalRegisterDocuments.js.map +1 -0
- package/dist/healthdataSocrataCatalogue.d.ts +4 -0
- package/dist/healthdataSocrataCatalogue.d.ts.map +1 -1
- package/dist/healthdataSocrataCatalogue.js +3 -0
- package/dist/healthdataSocrataCatalogue.js.map +1 -1
- package/dist/http.d.ts +14 -0
- package/dist/http.d.ts.map +1 -1
- package/dist/http.js +29 -1
- package/dist/http.js.map +1 -1
- package/dist/illinoisOpenDataCatalogue.d.ts +42 -0
- package/dist/illinoisOpenDataCatalogue.d.ts.map +1 -0
- package/dist/illinoisOpenDataCatalogue.js +45 -0
- package/dist/illinoisOpenDataCatalogue.js.map +1 -0
- package/dist/index.d.ts +76 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +38 -0
- package/dist/index.js.map +1 -1
- package/dist/losAngelesOpenDataCatalogue.d.ts +42 -0
- package/dist/losAngelesOpenDataCatalogue.d.ts.map +1 -0
- package/dist/losAngelesOpenDataCatalogue.js +48 -0
- package/dist/losAngelesOpenDataCatalogue.js.map +1 -0
- package/dist/michiganOpenDataCatalogue.d.ts +42 -0
- package/dist/michiganOpenDataCatalogue.d.ts.map +1 -0
- package/dist/michiganOpenDataCatalogue.js +45 -0
- package/dist/michiganOpenDataCatalogue.js.map +1 -0
- package/dist/newJerseyOpenDataCatalogue.d.ts +42 -0
- package/dist/newJerseyOpenDataCatalogue.d.ts.map +1 -0
- package/dist/newJerseyOpenDataCatalogue.js +48 -0
- package/dist/newJerseyOpenDataCatalogue.js.map +1 -0
- package/dist/nextBusTransitVehicles.d.ts.map +1 -1
- package/dist/nextBusTransitVehicles.js +14 -3
- package/dist/nextBusTransitVehicles.js.map +1 -1
- package/dist/nhtsaVehicleModels.d.ts +74 -0
- package/dist/nhtsaVehicleModels.d.ts.map +1 -0
- package/dist/nhtsaVehicleModels.js +139 -0
- package/dist/nhtsaVehicleModels.js.map +1 -0
- package/dist/nihReporterProjects.d.ts +98 -0
- package/dist/nihReporterProjects.d.ts.map +1 -0
- package/dist/nihReporterProjects.js +188 -0
- package/dist/nihReporterProjects.js.map +1 -0
- package/dist/nwsPointForecast.d.ts.map +1 -1
- package/dist/nwsPointForecast.js +13 -2
- package/dist/nwsPointForecast.js.map +1 -1
- package/dist/nyStateOpenDataCatalogue.d.ts +91 -0
- package/dist/nyStateOpenDataCatalogue.d.ts.map +1 -0
- package/dist/nyStateOpenDataCatalogue.js +133 -0
- package/dist/nyStateOpenDataCatalogue.js.map +1 -0
- package/dist/nycOpenDataCatalogue.d.ts +4 -0
- package/dist/nycOpenDataCatalogue.d.ts.map +1 -1
- package/dist/nycOpenDataCatalogue.js +3 -0
- package/dist/nycOpenDataCatalogue.js.map +1 -1
- package/dist/oregonOpenDataCatalogue.d.ts +42 -0
- package/dist/oregonOpenDataCatalogue.d.ts.map +1 -0
- package/dist/oregonOpenDataCatalogue.js +45 -0
- package/dist/oregonOpenDataCatalogue.js.map +1 -0
- package/dist/pennsylvaniaOpenDataCatalogue.d.ts +42 -0
- package/dist/pennsylvaniaOpenDataCatalogue.d.ts.map +1 -0
- package/dist/pennsylvaniaOpenDataCatalogue.js +48 -0
- package/dist/pennsylvaniaOpenDataCatalogue.js.map +1 -0
- package/dist/registry.d.ts +2 -1
- package/dist/registry.d.ts.map +1 -1
- package/dist/registry.js +51 -3
- package/dist/registry.js.map +1 -1
- package/dist/sanFranciscoOpenDataCatalogue.d.ts +42 -0
- package/dist/sanFranciscoOpenDataCatalogue.d.ts.map +1 -0
- package/dist/sanFranciscoOpenDataCatalogue.js +48 -0
- package/dist/sanFranciscoOpenDataCatalogue.js.map +1 -0
- package/dist/seattleOpenDataCatalogue.d.ts +91 -0
- package/dist/seattleOpenDataCatalogue.d.ts.map +1 -0
- package/dist/seattleOpenDataCatalogue.js +133 -0
- package/dist/seattleOpenDataCatalogue.js.map +1 -0
- package/dist/socrataCatalogue.d.ts +121 -0
- package/dist/socrataCatalogue.d.ts.map +1 -0
- package/dist/socrataCatalogue.js +151 -0
- package/dist/socrataCatalogue.js.map +1 -0
- package/dist/socrataCatalogueMetadata.d.ts +19 -0
- package/dist/socrataCatalogueMetadata.d.ts.map +1 -0
- package/dist/socrataCatalogueMetadata.js +40 -0
- package/dist/socrataCatalogueMetadata.js.map +1 -0
- package/dist/texasOpenDataCatalogue.d.ts +91 -0
- package/dist/texasOpenDataCatalogue.d.ts.map +1 -0
- package/dist/texasOpenDataCatalogue.js +133 -0
- package/dist/texasOpenDataCatalogue.js.map +1 -0
- package/dist/types.d.ts +2 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/vermontOpenDataCatalogue.d.ts +42 -0
- package/dist/vermontOpenDataCatalogue.d.ts.map +1 -0
- package/dist/vermontOpenDataCatalogue.js +45 -0
- package/dist/vermontOpenDataCatalogue.js.map +1 -0
- package/dist/washingtonStateOpenDataCatalogue.d.ts +42 -0
- package/dist/washingtonStateOpenDataCatalogue.d.ts.map +1 -0
- package/dist/washingtonStateOpenDataCatalogue.js +48 -0
- package/dist/washingtonStateOpenDataCatalogue.js.map +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { UsSourceParseError } from './errors.js';
|
|
3
|
+
import { readFixtureJson } from './fixtures.js';
|
|
4
|
+
import { httpGet } from './http.js';
|
|
5
|
+
import { parseSocrataCatalogueMetadata } from './socrataCatalogueMetadata.js';
|
|
6
|
+
/** Default number of catalogue entries one request reads. */
|
|
7
|
+
export const SOCRATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** Category bucket used when a catalogue entry carries no category. */
|
|
9
|
+
export const SOCRATA_UNCATEGORISED = 'Uncategorised';
|
|
10
|
+
// The discovery API sends a bare JSON array. A value the catalogue does not
|
|
11
|
+
// carry arrives as null, and every count arrives as a number.
|
|
12
|
+
const SOCRATA_CATALOGUE_ENTRY_SCHEMA = z.object({
|
|
13
|
+
id: z.string().optional(),
|
|
14
|
+
name: z.string().optional(),
|
|
15
|
+
assetType: z.string().optional(),
|
|
16
|
+
category: z.string().nullable().optional(),
|
|
17
|
+
viewType: z.string().optional(),
|
|
18
|
+
createdAt: z.number().optional(),
|
|
19
|
+
viewLastModified: z.number().optional(),
|
|
20
|
+
viewCount: z.number().optional(),
|
|
21
|
+
downloadCount: z.number().optional(),
|
|
22
|
+
metadata: z.unknown().optional(),
|
|
23
|
+
});
|
|
24
|
+
const SOCRATA_CATALOGUE_PAYLOAD_SCHEMA = z.array(SOCRATA_CATALOGUE_ENTRY_SCHEMA);
|
|
25
|
+
/** Counts entries per category, most entries first, ties by name. */
|
|
26
|
+
function countSocrataCategories(entries, uncategorisedLabel) {
|
|
27
|
+
const counts = new Map();
|
|
28
|
+
for (const entry of entries) {
|
|
29
|
+
const category = entry.category ?? uncategorisedLabel;
|
|
30
|
+
counts.set(category, (counts.get(category) ?? 0) + 1);
|
|
31
|
+
}
|
|
32
|
+
return [...counts.entries()]
|
|
33
|
+
.sort(([leftCategory, leftCount], [rightCategory, rightCount]) => leftCount === rightCount ? leftCategory.localeCompare(rightCategory) : rightCount - leftCount)
|
|
34
|
+
.map(([category, entryCount]) => ({ category, entryCount }));
|
|
35
|
+
}
|
|
36
|
+
/** Picks the entry with the latest creation time, ties to the earlier entry. */
|
|
37
|
+
function newestSocrataEntry(sourceId, entries) {
|
|
38
|
+
const first = entries[0];
|
|
39
|
+
if (first === undefined) {
|
|
40
|
+
throw new UsSourceParseError(sourceId, 'No entries to summarise');
|
|
41
|
+
}
|
|
42
|
+
return entries.reduce((best, entry) => (entry.createdAtEpochSeconds > best.createdAtEpochSeconds ? entry : best), first);
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Builds the request URL for one page of a Socrata catalogue.
|
|
46
|
+
*
|
|
47
|
+
* The endpoint is keyless. The service answers the whole catalogue by default,
|
|
48
|
+
* so the adapter asks for a short page unless the caller sets a cap.
|
|
49
|
+
*
|
|
50
|
+
* @param apiBase - the Socrata discovery API URL, without query parameters
|
|
51
|
+
* @param options - an optional entry cap
|
|
52
|
+
* @returns the full request URL
|
|
53
|
+
*/
|
|
54
|
+
export function buildSocrataCatalogueUrl(apiBase, options) {
|
|
55
|
+
const params = new URLSearchParams({
|
|
56
|
+
limit: String(options?.limit ?? SOCRATA_CATALOGUE_LIMIT),
|
|
57
|
+
});
|
|
58
|
+
return `${apiBase}?${params.toString()}`;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Parses one page of a Socrata catalogue into entries and topic counts.
|
|
62
|
+
*
|
|
63
|
+
* An entry without an id, a name, or a creation time is a changed shape and
|
|
64
|
+
* stops the parse. An empty page also stops the parse, because a catalogue
|
|
65
|
+
* with no entries is not a usable answer.
|
|
66
|
+
*
|
|
67
|
+
* @param sourceId - adapter id used in parse errors
|
|
68
|
+
* @param payload - the JSON body from the Socrata discovery API
|
|
69
|
+
* @param options - an optional category label for entries without one
|
|
70
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
71
|
+
*/
|
|
72
|
+
export function parseSocrataCataloguePayload(sourceId, payload, options) {
|
|
73
|
+
const parsed = SOCRATA_CATALOGUE_PAYLOAD_SCHEMA.safeParse(payload);
|
|
74
|
+
if (!parsed.success) {
|
|
75
|
+
throw new UsSourceParseError(sourceId, parsed.error.message);
|
|
76
|
+
}
|
|
77
|
+
if (parsed.data.length === 0) {
|
|
78
|
+
throw new UsSourceParseError(sourceId, 'The response carried no entries');
|
|
79
|
+
}
|
|
80
|
+
const entries = parsed.data.map((row) => {
|
|
81
|
+
if (row.id === undefined || row.name === undefined || row.createdAt === undefined) {
|
|
82
|
+
throw new UsSourceParseError(sourceId, 'A catalogue entry arrived without its id, name, or creation time');
|
|
83
|
+
}
|
|
84
|
+
return {
|
|
85
|
+
id: row.id,
|
|
86
|
+
name: row.name,
|
|
87
|
+
assetType: row.assetType ?? '',
|
|
88
|
+
category: row.category ?? null,
|
|
89
|
+
viewType: row.viewType ?? '',
|
|
90
|
+
createdAtEpochSeconds: row.createdAt,
|
|
91
|
+
viewLastModifiedEpochSeconds: row.viewLastModified ?? row.createdAt,
|
|
92
|
+
viewCount: row.viewCount ?? 0,
|
|
93
|
+
downloadCount: row.downloadCount ?? 0,
|
|
94
|
+
...parseSocrataCatalogueMetadata(row.metadata),
|
|
95
|
+
};
|
|
96
|
+
});
|
|
97
|
+
return {
|
|
98
|
+
entries,
|
|
99
|
+
entryCount: entries.length,
|
|
100
|
+
datasetCount: entries.filter((entry) => entry.assetType === 'dataset').length,
|
|
101
|
+
newestEntry: newestSocrataEntry(sourceId, entries),
|
|
102
|
+
categories: countSocrataCategories(entries, options?.uncategorisedLabel ?? SOCRATA_UNCATEGORISED),
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Reads one page of a Socrata catalogue.
|
|
107
|
+
*
|
|
108
|
+
* One keyless request covers the page and reports what the portal holds,
|
|
109
|
+
* without reading any dataset's rows.
|
|
110
|
+
*
|
|
111
|
+
* @param source - source id, API base, and optional defaults
|
|
112
|
+
* @param options - an optional entry cap and fetch stub
|
|
113
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
114
|
+
*/
|
|
115
|
+
export async function fetchSocrataCatalogue(source, options) {
|
|
116
|
+
const fetchImpl = options?.fetchImpl ?? globalThis.fetch;
|
|
117
|
+
const limit = options?.limit ?? source.limit;
|
|
118
|
+
const url = buildSocrataCatalogueUrl(source.apiBase, limit === undefined ? {} : { limit });
|
|
119
|
+
const response = await httpGet(source.sourceId, url, { fetchImpl });
|
|
120
|
+
return parseSocrataCataloguePayload(source.sourceId, await response.json(), source.uncategorisedLabel === undefined ? {} : { uncategorisedLabel: source.uncategorisedLabel });
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* Creates an adapter around one Socrata catalogue endpoint.
|
|
124
|
+
*
|
|
125
|
+
* @param config - source id, endpoint, fixture, and the text the API shows
|
|
126
|
+
* @returns an adapter that fetches, parses, and loads the committed snapshot
|
|
127
|
+
*/
|
|
128
|
+
export function createSocrataCatalogueAdapter(config) {
|
|
129
|
+
const source = {
|
|
130
|
+
sourceId: config.sourceId,
|
|
131
|
+
apiBase: config.apiBase,
|
|
132
|
+
...(config.limit === undefined ? {} : { limit: config.limit }),
|
|
133
|
+
...(config.uncategorisedLabel === undefined
|
|
134
|
+
? {}
|
|
135
|
+
: { uncategorisedLabel: config.uncategorisedLabel }),
|
|
136
|
+
};
|
|
137
|
+
return {
|
|
138
|
+
id: config.sourceId,
|
|
139
|
+
name: config.name,
|
|
140
|
+
auth: 'none',
|
|
141
|
+
description: config.description,
|
|
142
|
+
fetchLive: async (options) => fetchSocrataCatalogue(source, options?.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }),
|
|
143
|
+
parse: (payload) => parseSocrataCataloguePayload(source.sourceId, payload, source.uncategorisedLabel === undefined
|
|
144
|
+
? {}
|
|
145
|
+
: { uncategorisedLabel: source.uncategorisedLabel }),
|
|
146
|
+
loadFixture: () => parseSocrataCataloguePayload(source.sourceId, readFixtureJson(config.fixtureFilename), source.uncategorisedLabel === undefined
|
|
147
|
+
? {}
|
|
148
|
+
: { uncategorisedLabel: source.uncategorisedLabel }),
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
//# sourceMappingURL=socrataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"socrataCatalogue.js","sourceRoot":"","sources":["../src/socrataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAC;AAG9E,6DAA6D;AAC7D,MAAM,CAAC,MAAM,uBAAuB,GAAG,EAAE,CAAC;AAE1C,uEAAuE;AACvE,MAAM,CAAC,MAAM,qBAAqB,GAAG,eAAe,CAAC;AAyErD,4EAA4E;AAC5E,8DAA8D;AAC9D,MAAM,8BAA8B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAEH,MAAM,gCAAgC,GAAG,CAAC,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;AAEjF,qEAAqE;AACrE,SAAS,sBAAsB,CAC7B,OAAgC,EAChC,kBAA0B;IAE1B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,kBAAkB,CAAC;QACtD,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,CAC/D,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAC9F;SACA,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,gFAAgF;AAChF,SAAS,kBAAkB,CACzB,QAAgB,EAChB,OAAgC;IAEhC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,yBAAyB,CAAC,CAAC;IACpE,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAC1F,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAC,OAAe,EAAE,OAA4B;IACpF,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,uBAAuB,CAAC;KACzD,CAAC,CAAC;IACH,OAAO,GAAG,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC3C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,4BAA4B,CAC1C,QAAgB,EAChB,OAAgB,EAChB,OAAyC;IAEzC,MAAM,MAAM,GAAG,gCAAgC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACnE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,kBAAkB,CAAC,QAAQ,EAAE,iCAAiC,CAAC,CAAC;IAC5E,CAAC;IAED,MAAM,OAAO,GAA4B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QAC/D,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAClF,MAAM,IAAI,kBAAkB,CAC1B,QAAQ,EACR,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,4BAA4B,EAAE,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS;YACnE,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;YAC7B,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC;YACrC,GAAG,6BAA6B,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7E,WAAW,EAAE,kBAAkB,CAAC,QAAQ,EAAE,OAAO,CAAC;QAClD,UAAU,EAAE,sBAAsB,CAChC,OAAO,EACP,OAAO,EAAE,kBAAkB,IAAI,qBAAqB,CACrD;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,MAA8B,EAC9B,OAAiE;IAEjE,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACzD,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,IAAI,MAAM,CAAC,KAAK,CAAC;IAC7C,MAAM,GAAG,GAAG,wBAAwB,CAAC,MAAM,CAAC,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;IAC3F,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACpE,OAAO,4BAA4B,CACjC,MAAM,CAAC,QAAQ,EACf,MAAM,QAAQ,CAAC,IAAI,EAAE,EACrB,MAAM,CAAC,kBAAkB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CACjG,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,6BAA6B,CAC3C,MAAqC;IAErC,MAAM,MAAM,GAA2B;QACrC,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,GAAG,CAAC,MAAM,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC;QAC9D,GAAG,CAAC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YACzC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CAAC;KACvD,CAAC;IACF,OAAO;QACL,EAAE,EAAE,MAAM,CAAC,QAAQ;QACnB,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC3B,qBAAqB,CACnB,MAAM,EACN,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACzE;QACH,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CACjB,4BAA4B,CAC1B,MAAM,CAAC,QAAQ,EACf,OAAO,EACP,MAAM,CAAC,kBAAkB,KAAK,SAAS;YACrC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CACtD;QACH,WAAW,EAAE,GAAG,EAAE,CAChB,4BAA4B,CAC1B,MAAM,CAAC,QAAQ,EACf,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,EACvC,MAAM,CAAC,kBAAkB,KAAK,SAAS;YACrC,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,EAAE,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,EAAE,CACtD;KACJ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/** Source-supplied licence and attribution metadata for one Socrata entry. */
|
|
2
|
+
export interface SocrataCatalogueMetadata {
|
|
3
|
+
/** Licence URL the portal attaches to the entry, or "" when it supplies none. */
|
|
4
|
+
licenceUrl: string;
|
|
5
|
+
/** Publishing organisation the portal attaches to the entry, or "" when it supplies none. */
|
|
6
|
+
attribution: string;
|
|
7
|
+
}
|
|
8
|
+
/**
|
|
9
|
+
* Reads the licence and attribution values Socrata attaches to a catalogue entry.
|
|
10
|
+
*
|
|
11
|
+
* Portals use "Common Core" for older entries and "Common Core 3" for newer
|
|
12
|
+
* ones. A missing or malformed block returns empty strings, because the portal
|
|
13
|
+
* did not supply the value rather than the adapter knowing better.
|
|
14
|
+
*
|
|
15
|
+
* @param metadata - the entry's Socrata metadata block
|
|
16
|
+
* @returns the source-supplied licence URL and publisher
|
|
17
|
+
*/
|
|
18
|
+
export declare function parseSocrataCatalogueMetadata(metadata: unknown): SocrataCatalogueMetadata;
|
|
19
|
+
//# sourceMappingURL=socrataCatalogueMetadata.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"socrataCatalogueMetadata.d.ts","sourceRoot":"","sources":["../src/socrataCatalogueMetadata.ts"],"names":[],"mappings":"AAEA,8EAA8E;AAC9E,MAAM,WAAW,wBAAwB;IACvC,iFAAiF;IACjF,UAAU,EAAE,MAAM,CAAC;IACnB,6FAA6F;IAC7F,WAAW,EAAE,MAAM,CAAC;CACrB;AAqBD;;;;;;;;;GASG;AACH,wBAAgB,6BAA6B,CAAC,QAAQ,EAAE,OAAO,GAAG,wBAAwB,CAUzF"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
const SOCRATA_CATALOGUE_METADATA_SCHEMA = z
|
|
3
|
+
.object({
|
|
4
|
+
custom_fields: z.record(z.string(), z.record(z.string(), z.unknown())).nullable().optional(),
|
|
5
|
+
})
|
|
6
|
+
.nullable()
|
|
7
|
+
.optional();
|
|
8
|
+
/**
|
|
9
|
+
* Reads one string field from the portal's Common Core metadata groups.
|
|
10
|
+
*
|
|
11
|
+
* @param groups - the portal's custom field groups
|
|
12
|
+
* @param key - the source field name, such as License or Publisher
|
|
13
|
+
* @returns the trimmed value, or "" when the source did not supply a string
|
|
14
|
+
*/
|
|
15
|
+
function readCommonCoreText(groups, key) {
|
|
16
|
+
const value = groups['Common Core']?.[key] ?? groups['Common Core 3']?.[key];
|
|
17
|
+
return typeof value === 'string' ? value.trim() : '';
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Reads the licence and attribution values Socrata attaches to a catalogue entry.
|
|
21
|
+
*
|
|
22
|
+
* Portals use "Common Core" for older entries and "Common Core 3" for newer
|
|
23
|
+
* ones. A missing or malformed block returns empty strings, because the portal
|
|
24
|
+
* did not supply the value rather than the adapter knowing better.
|
|
25
|
+
*
|
|
26
|
+
* @param metadata - the entry's Socrata metadata block
|
|
27
|
+
* @returns the source-supplied licence URL and publisher
|
|
28
|
+
*/
|
|
29
|
+
export function parseSocrataCatalogueMetadata(metadata) {
|
|
30
|
+
const parsed = SOCRATA_CATALOGUE_METADATA_SCHEMA.safeParse(metadata);
|
|
31
|
+
if (!parsed.success || parsed.data?.custom_fields == null) {
|
|
32
|
+
return { licenceUrl: '', attribution: '' };
|
|
33
|
+
}
|
|
34
|
+
const groups = parsed.data.custom_fields;
|
|
35
|
+
return {
|
|
36
|
+
licenceUrl: readCommonCoreText(groups, 'License'),
|
|
37
|
+
attribution: readCommonCoreText(groups, 'Publisher'),
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
//# sourceMappingURL=socrataCatalogueMetadata.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"socrataCatalogueMetadata.js","sourceRoot":"","sources":["../src/socrataCatalogueMetadata.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,iCAAiC,GAAG,CAAC;KACxC,MAAM,CAAC;IACN,aAAa,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;CAC7F,CAAC;KACD,QAAQ,EAAE;KACV,QAAQ,EAAE,CAAC;AAEd;;;;;;GAMG;AACH,SAAS,kBAAkB,CAAC,MAA+C,EAAE,GAAW;IACtF,MAAM,KAAK,GAAG,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;AACvD,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,6BAA6B,CAAC,QAAiB;IAC7D,MAAM,MAAM,GAAG,iCAAiC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IACrE,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,MAAM,CAAC,IAAI,EAAE,aAAa,IAAI,IAAI,EAAE,CAAC;QAC1D,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC;IAC7C,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC;IACzC,OAAO;QACL,UAAU,EAAE,kBAAkB,CAAC,MAAM,EAAE,SAAS,CAAC;QACjD,WAAW,EAAE,kBAAkB,CAAC,MAAM,EAAE,WAAW,CAAC;KACrD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { UsDataAdapter } from './types.js';
|
|
2
|
+
/** Adapter id for the Texas Open Data catalogue source. */
|
|
3
|
+
export declare const TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID = "texas-open-data-catalogue";
|
|
4
|
+
/** Base URL for the Texas Open Data Socrata catalogue. */
|
|
5
|
+
export declare const TEXAS_OPEN_DATA_API_BASE = "https://data.texas.gov/api/views.json";
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export declare const TEXAS_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** One catalogue entry from the Texas Socrata discovery API. */
|
|
9
|
+
export interface TexasOpenDataCatalogueEntry {
|
|
10
|
+
/** Four-by-four Socrata dataset id, e.g. "nqfs-mwbh". */
|
|
11
|
+
id: string;
|
|
12
|
+
/** Human-readable title of the dataset, chart, story, map, or file. */
|
|
13
|
+
name: string;
|
|
14
|
+
/** What the entry is: "dataset", "href", "map", or "measure". */
|
|
15
|
+
assetType: string;
|
|
16
|
+
/** State topic the entry belongs to, or null when the catalogue lists none. */
|
|
17
|
+
category: string | null;
|
|
18
|
+
/** How the entry is rendered, e.g. "tabular" or "href". */
|
|
19
|
+
viewType: string;
|
|
20
|
+
/** When the entry was created, in seconds since the Unix epoch. */
|
|
21
|
+
createdAtEpochSeconds: number;
|
|
22
|
+
/** When the entry was last modified, in seconds since the Unix epoch. */
|
|
23
|
+
viewLastModifiedEpochSeconds: number;
|
|
24
|
+
/** How many times the entry has been viewed. */
|
|
25
|
+
viewCount: number;
|
|
26
|
+
/** How many times the entry has been downloaded. */
|
|
27
|
+
downloadCount: number;
|
|
28
|
+
/** Licence text or URL the portal attaches, or "" when it supplies none. */
|
|
29
|
+
licenceUrl: string;
|
|
30
|
+
/** Publishing organisation the portal attaches, or "" when it supplies none. */
|
|
31
|
+
attribution: string;
|
|
32
|
+
}
|
|
33
|
+
/** One state topic and how many catalogue entries carry it. */
|
|
34
|
+
export interface TexasOpenDataCategoryCount {
|
|
35
|
+
category: string;
|
|
36
|
+
entryCount: number;
|
|
37
|
+
}
|
|
38
|
+
/** One page of the Texas catalogue, folded for a story. */
|
|
39
|
+
export interface TexasOpenDataCatalogue {
|
|
40
|
+
/** One entry per catalogue item in the page. */
|
|
41
|
+
entries: TexasOpenDataCatalogueEntry[];
|
|
42
|
+
entryCount: number;
|
|
43
|
+
/** Entries whose asset type is "dataset". */
|
|
44
|
+
datasetCount: number;
|
|
45
|
+
/** Entry with the latest creation time. Ties go to the earlier list entry. */
|
|
46
|
+
newestEntry: TexasOpenDataCatalogueEntry;
|
|
47
|
+
/**
|
|
48
|
+
* Categories in the page, most entries first, ties broken by name. An entry
|
|
49
|
+
* the catalogue lists without a category is counted under "Uncategorised".
|
|
50
|
+
*/
|
|
51
|
+
categories: TexasOpenDataCategoryCount[];
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Builds the request URL for one page of the Texas catalogue.
|
|
55
|
+
*
|
|
56
|
+
* The endpoint is keyless. The service answers the whole catalogue by default,
|
|
57
|
+
* so the adapter asks for a short page.
|
|
58
|
+
*
|
|
59
|
+
* @param options - an optional entry cap
|
|
60
|
+
* @returns the full request URL
|
|
61
|
+
*/
|
|
62
|
+
export declare function buildTexasOpenDataCatalogueUrl(options?: {
|
|
63
|
+
limit?: number;
|
|
64
|
+
}): string;
|
|
65
|
+
/**
|
|
66
|
+
* Parses one page of the Texas catalogue into the entries and topic counts.
|
|
67
|
+
*
|
|
68
|
+
* An entry without an id, a name, or a creation time is a changed shape and
|
|
69
|
+
* stops the parse. An empty page also stops the parse, because a catalogue
|
|
70
|
+
* with no entries is not a usable answer.
|
|
71
|
+
*
|
|
72
|
+
* @param payload - the JSON body from the Texas Socrata discovery API
|
|
73
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
74
|
+
*/
|
|
75
|
+
export declare function parseTexasOpenDataCataloguePayload(payload: unknown): TexasOpenDataCatalogue;
|
|
76
|
+
/**
|
|
77
|
+
* Reads one page of the Texas Open Data catalogue.
|
|
78
|
+
*
|
|
79
|
+
* One keyless request covers the page and reports what the state portal holds,
|
|
80
|
+
* without reading any dataset's rows.
|
|
81
|
+
*
|
|
82
|
+
* @param options - an optional entry cap and fetch stub
|
|
83
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
84
|
+
*/
|
|
85
|
+
export declare function fetchTexasOpenDataCatalogue(options?: {
|
|
86
|
+
limit?: number;
|
|
87
|
+
fetchImpl?: typeof globalThis.fetch;
|
|
88
|
+
}): Promise<TexasOpenDataCatalogue>;
|
|
89
|
+
/** Texas Open Data catalogue of datasets, maps, charts, and linked resources, keyless. */
|
|
90
|
+
export declare const texasOpenDataCatalogueAdapter: UsDataAdapter<TexasOpenDataCatalogue>;
|
|
91
|
+
//# sourceMappingURL=texasOpenDataCatalogue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"texasOpenDataCatalogue.d.ts","sourceRoot":"","sources":["../src/texasOpenDataCatalogue.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,2DAA2D;AAC3D,eAAO,MAAM,mCAAmC,8BAA8B,CAAC;AAE/E,0DAA0D;AAC1D,eAAO,MAAM,wBAAwB,0CAA0C,CAAC;AAEhF,oDAAoD;AACpD,eAAO,MAAM,+BAA+B,KAAK,CAAC;AAElD,gEAAgE;AAChE,MAAM,WAAW,2BAA2B;IAC1C,yDAAyD;IACzD,EAAE,EAAE,MAAM,CAAC;IACX,uEAAuE;IACvE,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,2DAA2D;IAC3D,QAAQ,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,qBAAqB,EAAE,MAAM,CAAC;IAC9B,yEAAyE;IACzE,4BAA4B,EAAE,MAAM,CAAC;IACrC,gDAAgD;IAChD,SAAS,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,aAAa,EAAE,MAAM,CAAC;IACtB,4EAA4E;IAC5E,UAAU,EAAE,MAAM,CAAC;IACnB,gFAAgF;IAChF,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,+DAA+D;AAC/D,MAAM,WAAW,0BAA0B;IACzC,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,2DAA2D;AAC3D,MAAM,WAAW,sBAAsB;IACrC,gDAAgD;IAChD,OAAO,EAAE,2BAA2B,EAAE,CAAC;IACvC,UAAU,EAAE,MAAM,CAAC;IACnB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;IACrB,8EAA8E;IAC9E,WAAW,EAAE,2BAA2B,CAAC;IACzC;;;OAGG;IACH,UAAU,EAAE,0BAA0B,EAAE,CAAC;CAC1C;AAuDD;;;;;;;;GAQG;AACH,wBAAgB,8BAA8B,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAKnF;AAED;;;;;;;;;GASG;AACH,wBAAgB,kCAAkC,CAAC,OAAO,EAAE,OAAO,GAAG,sBAAsB,CAwC3F;AAED;;;;;;;;GAQG;AACH,wBAAsB,2BAA2B,CAAC,OAAO,CAAC,EAAE;IAC1D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAOlC;AAED,0FAA0F;AAC1F,eAAO,MAAM,6BAA6B,EAAE,aAAa,CAAC,sBAAsB,CAa/E,CAAC"}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { UsSourceParseError } from './errors.js';
|
|
3
|
+
import { readFixtureJson } from './fixtures.js';
|
|
4
|
+
import { httpGet } from './http.js';
|
|
5
|
+
import { parseSocrataCatalogueMetadata } from './socrataCatalogueMetadata.js';
|
|
6
|
+
/** Adapter id for the Texas Open Data catalogue source. */
|
|
7
|
+
export const TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID = 'texas-open-data-catalogue';
|
|
8
|
+
/** Base URL for the Texas Open Data Socrata catalogue. */
|
|
9
|
+
export const TEXAS_OPEN_DATA_API_BASE = 'https://data.texas.gov/api/views.json';
|
|
10
|
+
/** How many catalogue entries one request reads. */
|
|
11
|
+
export const TEXAS_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
12
|
+
/** Committed snapshot, so a build works when the Texas host is unreachable. */
|
|
13
|
+
const TEXAS_OPEN_DATA_FIXTURE_FILENAME = 'texas-open-data-catalogue-2026-11-08.json';
|
|
14
|
+
/** Category bucket the adapter uses for an entry the catalogue lists without one. */
|
|
15
|
+
const TEXAS_OPEN_DATA_UNCATEGORISED = 'Uncategorised';
|
|
16
|
+
// The discovery API sends a bare JSON array. A value the catalogue does not
|
|
17
|
+
// carry arrives as null, and every count arrives as a number.
|
|
18
|
+
const TEXAS_OPEN_DATA_ENTRY_SCHEMA = z.object({
|
|
19
|
+
id: z.string().optional(),
|
|
20
|
+
name: z.string().optional(),
|
|
21
|
+
assetType: z.string().optional(),
|
|
22
|
+
category: z.string().nullable().optional(),
|
|
23
|
+
viewType: z.string().optional(),
|
|
24
|
+
createdAt: z.number().optional(),
|
|
25
|
+
viewLastModified: z.number().optional(),
|
|
26
|
+
viewCount: z.number().optional(),
|
|
27
|
+
downloadCount: z.number().optional(),
|
|
28
|
+
metadata: z.unknown().optional(),
|
|
29
|
+
});
|
|
30
|
+
const TEXAS_OPEN_DATA_PAYLOAD_SCHEMA = z.array(TEXAS_OPEN_DATA_ENTRY_SCHEMA);
|
|
31
|
+
/** Counts entries per category, most entries first, ties by name. */
|
|
32
|
+
function countTexasOpenDataCategories(entries) {
|
|
33
|
+
const counts = new Map();
|
|
34
|
+
for (const entry of entries) {
|
|
35
|
+
const category = entry.category ?? TEXAS_OPEN_DATA_UNCATEGORISED;
|
|
36
|
+
counts.set(category, (counts.get(category) ?? 0) + 1);
|
|
37
|
+
}
|
|
38
|
+
return [...counts.entries()]
|
|
39
|
+
.sort(([leftCategory, leftCount], [rightCategory, rightCount]) => leftCount === rightCount ? leftCategory.localeCompare(rightCategory) : rightCount - leftCount)
|
|
40
|
+
.map(([category, entryCount]) => ({ category, entryCount }));
|
|
41
|
+
}
|
|
42
|
+
/** Picks the entry with the latest creation time, ties to the earlier entry. */
|
|
43
|
+
function newestTexasOpenDataEntry(entries) {
|
|
44
|
+
const first = entries[0];
|
|
45
|
+
if (first === undefined) {
|
|
46
|
+
throw new UsSourceParseError(TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID, 'No entries to summarise');
|
|
47
|
+
}
|
|
48
|
+
return entries.reduce((best, entry) => (entry.createdAtEpochSeconds > best.createdAtEpochSeconds ? entry : best), first);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* Builds the request URL for one page of the Texas catalogue.
|
|
52
|
+
*
|
|
53
|
+
* The endpoint is keyless. The service answers the whole catalogue by default,
|
|
54
|
+
* so the adapter asks for a short page.
|
|
55
|
+
*
|
|
56
|
+
* @param options - an optional entry cap
|
|
57
|
+
* @returns the full request URL
|
|
58
|
+
*/
|
|
59
|
+
export function buildTexasOpenDataCatalogueUrl(options) {
|
|
60
|
+
const params = new URLSearchParams({
|
|
61
|
+
limit: String(options?.limit ?? TEXAS_OPEN_DATA_CATALOGUE_LIMIT),
|
|
62
|
+
});
|
|
63
|
+
return `${TEXAS_OPEN_DATA_API_BASE}?${params.toString()}`;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Parses one page of the Texas catalogue into the entries and topic counts.
|
|
67
|
+
*
|
|
68
|
+
* An entry without an id, a name, or a creation time is a changed shape and
|
|
69
|
+
* stops the parse. An empty page also stops the parse, because a catalogue
|
|
70
|
+
* with no entries is not a usable answer.
|
|
71
|
+
*
|
|
72
|
+
* @param payload - the JSON body from the Texas Socrata discovery API
|
|
73
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
74
|
+
*/
|
|
75
|
+
export function parseTexasOpenDataCataloguePayload(payload) {
|
|
76
|
+
const parsed = TEXAS_OPEN_DATA_PAYLOAD_SCHEMA.safeParse(payload);
|
|
77
|
+
if (!parsed.success) {
|
|
78
|
+
throw new UsSourceParseError(TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID, parsed.error.message);
|
|
79
|
+
}
|
|
80
|
+
if (parsed.data.length === 0) {
|
|
81
|
+
throw new UsSourceParseError(TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID, 'The response carried no entries');
|
|
82
|
+
}
|
|
83
|
+
const entries = parsed.data.map((row) => {
|
|
84
|
+
if (row.id === undefined || row.name === undefined || row.createdAt === undefined) {
|
|
85
|
+
throw new UsSourceParseError(TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID, 'A catalogue entry arrived without its id, name, or creation time');
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
id: row.id,
|
|
89
|
+
name: row.name,
|
|
90
|
+
assetType: row.assetType ?? '',
|
|
91
|
+
category: row.category ?? null,
|
|
92
|
+
viewType: row.viewType ?? '',
|
|
93
|
+
createdAtEpochSeconds: row.createdAt,
|
|
94
|
+
viewLastModifiedEpochSeconds: row.viewLastModified ?? row.createdAt,
|
|
95
|
+
viewCount: row.viewCount ?? 0,
|
|
96
|
+
downloadCount: row.downloadCount ?? 0,
|
|
97
|
+
...parseSocrataCatalogueMetadata(row.metadata),
|
|
98
|
+
};
|
|
99
|
+
});
|
|
100
|
+
return {
|
|
101
|
+
entries,
|
|
102
|
+
entryCount: entries.length,
|
|
103
|
+
datasetCount: entries.filter((entry) => entry.assetType === 'dataset').length,
|
|
104
|
+
newestEntry: newestTexasOpenDataEntry(entries),
|
|
105
|
+
categories: countTexasOpenDataCategories(entries),
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Reads one page of the Texas Open Data catalogue.
|
|
110
|
+
*
|
|
111
|
+
* One keyless request covers the page and reports what the state portal holds,
|
|
112
|
+
* without reading any dataset's rows.
|
|
113
|
+
*
|
|
114
|
+
* @param options - an optional entry cap and fetch stub
|
|
115
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
116
|
+
*/
|
|
117
|
+
export async function fetchTexasOpenDataCatalogue(options) {
|
|
118
|
+
const fetchImpl = options?.fetchImpl ?? globalThis.fetch;
|
|
119
|
+
const url = buildTexasOpenDataCatalogueUrl(options?.limit === undefined ? {} : { limit: options.limit });
|
|
120
|
+
const response = await httpGet(TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID, url, { fetchImpl });
|
|
121
|
+
return parseTexasOpenDataCataloguePayload(await response.json());
|
|
122
|
+
}
|
|
123
|
+
/** Texas Open Data catalogue of datasets, maps, charts, and linked resources, keyless. */
|
|
124
|
+
export const texasOpenDataCatalogueAdapter = {
|
|
125
|
+
id: TEXAS_OPEN_DATA_CATALOGUE_SOURCE_ID,
|
|
126
|
+
name: 'Texas Open Data catalogue',
|
|
127
|
+
auth: 'none',
|
|
128
|
+
description: 'The Texas Open Data Socrata catalogue, one row per published dataset, map, chart, or linked resource, with its state topic, view count, and download count.',
|
|
129
|
+
fetchLive: async (options) => fetchTexasOpenDataCatalogue(options?.fetchImpl === undefined ? {} : { fetchImpl: options.fetchImpl }),
|
|
130
|
+
parse: (payload) => parseTexasOpenDataCataloguePayload(payload),
|
|
131
|
+
loadFixture: () => parseTexasOpenDataCataloguePayload(readFixtureJson(TEXAS_OPEN_DATA_FIXTURE_FILENAME)),
|
|
132
|
+
};
|
|
133
|
+
//# sourceMappingURL=texasOpenDataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"texasOpenDataCatalogue.js","sourceRoot":"","sources":["../src/texasOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,6BAA6B,EAAE,MAAM,+BAA+B,CAAC;AAG9E,2DAA2D;AAC3D,MAAM,CAAC,MAAM,mCAAmC,GAAG,2BAA2B,CAAC;AAE/E,0DAA0D;AAC1D,MAAM,CAAC,MAAM,wBAAwB,GAAG,uCAAuC,CAAC;AAEhF,oDAAoD;AACpD,MAAM,CAAC,MAAM,+BAA+B,GAAG,EAAE,CAAC;AAkDlD,+EAA+E;AAC/E,MAAM,gCAAgC,GAAG,2CAA2C,CAAC;AAErF,qFAAqF;AACrF,MAAM,6BAA6B,GAAG,eAAe,CAAC;AAEtD,4EAA4E;AAC5E,8DAA8D;AAC9D,MAAM,4BAA4B,GAAG,CAAC,CAAC,MAAM,CAAC;IAC5C,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC1C,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC/B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,gBAAgB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACvC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,aAAa,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACpC,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,QAAQ,EAAE;CACjC,CAAC,CAAC;AAEH,MAAM,8BAA8B,GAAG,CAAC,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;AAE7E,qEAAqE;AACrE,SAAS,4BAA4B,CACnC,OAAsC;IAEtC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,6BAA6B,CAAC;QACjE,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;SACzB,IAAI,CAAC,CAAC,CAAC,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,EAAE,CAC/D,SAAS,KAAK,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,aAAa,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,UAAU,GAAG,SAAS,CAC9F;SACA,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,gFAAgF;AAChF,SAAS,wBAAwB,CAC/B,OAAsC;IAEtC,MAAM,KAAK,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,IAAI,kBAAkB,CAAC,mCAAmC,EAAE,yBAAyB,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,OAAO,CAAC,MAAM,CACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,qBAAqB,GAAG,IAAI,CAAC,qBAAqB,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAC1F,KAAK,CACN,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,8BAA8B,CAAC,OAA4B;IACzE,MAAM,MAAM,GAAG,IAAI,eAAe,CAAC;QACjC,KAAK,EAAE,MAAM,CAAC,OAAO,EAAE,KAAK,IAAI,+BAA+B,CAAC;KACjE,CAAC,CAAC;IACH,OAAO,GAAG,wBAAwB,IAAI,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;AAC5D,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,kCAAkC,CAAC,OAAgB;IACjE,MAAM,MAAM,GAAG,8BAA8B,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,kBAAkB,CAAC,mCAAmC,EAAE,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC1F,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC7B,MAAM,IAAI,kBAAkB,CAC1B,mCAAmC,EACnC,iCAAiC,CAClC,CAAC;IACJ,CAAC;IAED,MAAM,OAAO,GAAkC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;QACrE,IAAI,GAAG,CAAC,EAAE,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YAClF,MAAM,IAAI,kBAAkB,CAC1B,mCAAmC,EACnC,kEAAkE,CACnE,CAAC;QACJ,CAAC;QACD,OAAO;YACL,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,EAAE;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;YAC9B,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,EAAE;YAC5B,qBAAqB,EAAE,GAAG,CAAC,SAAS;YACpC,4BAA4B,EAAE,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS;YACnE,SAAS,EAAE,GAAG,CAAC,SAAS,IAAI,CAAC;YAC7B,aAAa,EAAE,GAAG,CAAC,aAAa,IAAI,CAAC;YACrC,GAAG,6BAA6B,CAAC,GAAG,CAAC,QAAQ,CAAC;SAC/C,CAAC;IACJ,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,OAAO;QACP,UAAU,EAAE,OAAO,CAAC,MAAM;QAC1B,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,MAAM;QAC7E,WAAW,EAAE,wBAAwB,CAAC,OAAO,CAAC;QAC9C,UAAU,EAAE,4BAA4B,CAAC,OAAO,CAAC;KAClD,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,2BAA2B,CAAC,OAGjD;IACC,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,UAAU,CAAC,KAAK,CAAC;IACzD,MAAM,GAAG,GAAG,8BAA8B,CACxC,OAAO,EAAE,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAC7D,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,mCAAmC,EAAE,GAAG,EAAE,EAAE,SAAS,EAAE,CAAC,CAAC;IACxF,OAAO,kCAAkC,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,0FAA0F;AAC1F,MAAM,CAAC,MAAM,6BAA6B,GAA0C;IAClF,EAAE,EAAE,mCAAmC;IACvC,IAAI,EAAE,2BAA2B;IACjC,IAAI,EAAE,MAAM;IACZ,WAAW,EACT,6JAA6J;IAC/J,SAAS,EAAE,KAAK,EAAE,OAAO,EAAE,EAAE,CAC3B,2BAA2B,CACzB,OAAO,EAAE,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CACzE;IACH,KAAK,EAAE,CAAC,OAAO,EAAE,EAAE,CAAC,kCAAkC,CAAC,OAAO,CAAC;IAC/D,WAAW,EAAE,GAAG,EAAE,CAChB,kCAAkC,CAAC,eAAe,CAAC,gCAAgC,CAAC,CAAC;CACxF,CAAC"}
|
package/dist/types.d.ts
CHANGED
|
@@ -7,7 +7,8 @@ export interface UsFetchOptions {
|
|
|
7
7
|
}
|
|
8
8
|
/**
|
|
9
9
|
* One US data source behind a uniform interface: a live fetch, a strict
|
|
10
|
-
* parse, and a committed fixture
|
|
10
|
+
* parse, and a committed fixture for tests. A live fetch never falls back
|
|
11
|
+
* to the fixture.
|
|
11
12
|
*/
|
|
12
13
|
export interface UsDataAdapter<T> {
|
|
13
14
|
readonly id: string;
|
package/dist/types.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,CAAC;AAE1C,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC;AAED
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,8DAA8D;AAC9D,MAAM,MAAM,YAAY,GAAG,MAAM,GAAG,KAAK,CAAC;AAE1C,oDAAoD;AACpD,MAAM,WAAW,cAAc;IAC7B,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC;AAED;;;;GAIG;AACH,MAAM,WAAW,aAAa,CAAC,CAAC;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,SAAS,CAAC,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAChD,KAAK,CAAC,OAAO,EAAE,OAAO,GAAG,CAAC,CAAC;IAC3B,WAAW,IAAI,CAAC,CAAC;CAClB;AAED,8DAA8D;AAC9D,MAAM,WAAW,aAAa;IAC5B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,YAAY,CAAC;IACnB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type SocrataCatalogue, type SocrataCatalogueCategoryCount, type SocrataCatalogueEntry } from './socrataCatalogue.js';
|
|
2
|
+
/** Adapter id for the Vermont Open Data catalogue source. */
|
|
3
|
+
export declare const VERMONT_OPEN_DATA_CATALOGUE_SOURCE_ID = "vermont-open-data-catalogue";
|
|
4
|
+
/** Base URL for the Vermont Open Data Socrata catalogue. */
|
|
5
|
+
export declare const VERMONT_OPEN_DATA_API_BASE = "https://data.vermont.gov/api/views.json";
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export declare const VERMONT_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** One catalogue entry from the Vermont Socrata discovery API. */
|
|
9
|
+
export type VermontOpenDataCatalogueEntry = SocrataCatalogueEntry;
|
|
10
|
+
/** One state topic and how many catalogue entries carry it. */
|
|
11
|
+
export type VermontOpenDataCategoryCount = SocrataCatalogueCategoryCount;
|
|
12
|
+
/** One page of the Vermont catalogue, folded for a story. */
|
|
13
|
+
export type VermontOpenDataCatalogue = SocrataCatalogue;
|
|
14
|
+
/**
|
|
15
|
+
* Builds the request URL for one page of the Vermont catalogue.
|
|
16
|
+
*
|
|
17
|
+
* @param options - an optional entry cap
|
|
18
|
+
* @returns the full request URL
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildVermontOpenDataCatalogueUrl(options?: {
|
|
21
|
+
limit?: number;
|
|
22
|
+
}): string;
|
|
23
|
+
/**
|
|
24
|
+
* Parses one page of the Vermont catalogue into entries and topic counts.
|
|
25
|
+
*
|
|
26
|
+
* @param payload - the JSON body from the Vermont Socrata discovery API
|
|
27
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
28
|
+
*/
|
|
29
|
+
export declare function parseVermontOpenDataCataloguePayload(payload: unknown): VermontOpenDataCatalogue;
|
|
30
|
+
/**
|
|
31
|
+
* Reads one page of the Vermont Open Data catalogue.
|
|
32
|
+
*
|
|
33
|
+
* @param options - an optional entry cap and fetch stub
|
|
34
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
35
|
+
*/
|
|
36
|
+
export declare function fetchVermontOpenDataCatalogue(options?: {
|
|
37
|
+
limit?: number;
|
|
38
|
+
fetchImpl?: typeof globalThis.fetch;
|
|
39
|
+
}): Promise<VermontOpenDataCatalogue>;
|
|
40
|
+
/** Vermont Open Data catalogue of datasets and filtered views, keyless. */
|
|
41
|
+
export declare const vermontOpenDataCatalogueAdapter: import("./types.js").UsDataAdapter<SocrataCatalogue>;
|
|
42
|
+
//# sourceMappingURL=vermontOpenDataCatalogue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vermontOpenDataCatalogue.d.ts","sourceRoot":"","sources":["../src/vermontOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,qBAAqB,EAC3B,MAAM,uBAAuB,CAAC;AAE/B,6DAA6D;AAC7D,eAAO,MAAM,qCAAqC,gCAAgC,CAAC;AAEnF,4DAA4D;AAC5D,eAAO,MAAM,0BAA0B,4CAA4C,CAAC;AAEpF,oDAAoD;AACpD,eAAO,MAAM,iCAAiC,KAA0B,CAAC;AAEzE,kEAAkE;AAClE,MAAM,MAAM,6BAA6B,GAAG,qBAAqB,CAAC;AAElE,+DAA+D;AAC/D,MAAM,MAAM,4BAA4B,GAAG,6BAA6B,CAAC;AAEzE,6DAA6D;AAC7D,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAC;AAKxD;;;;;GAKG;AACH,wBAAgB,gCAAgC,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAErF;AAED;;;;;GAKG;AACH,wBAAgB,oCAAoC,CAAC,OAAO,EAAE,OAAO,GAAG,wBAAwB,CAE/F;AAED;;;;;GAKG;AACH,wBAAsB,6BAA6B,CAAC,OAAO,CAAC,EAAE;IAC5D,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAKpC;AAED,2EAA2E;AAC3E,eAAO,MAAM,+BAA+B,sDAO1C,CAAC"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { buildSocrataCatalogueUrl, createSocrataCatalogueAdapter, fetchSocrataCatalogue, parseSocrataCataloguePayload, SOCRATA_CATALOGUE_LIMIT, } from './socrataCatalogue.js';
|
|
2
|
+
/** Adapter id for the Vermont Open Data catalogue source. */
|
|
3
|
+
export const VERMONT_OPEN_DATA_CATALOGUE_SOURCE_ID = 'vermont-open-data-catalogue';
|
|
4
|
+
/** Base URL for the Vermont Open Data Socrata catalogue. */
|
|
5
|
+
export const VERMONT_OPEN_DATA_API_BASE = 'https://data.vermont.gov/api/views.json';
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export const VERMONT_OPEN_DATA_CATALOGUE_LIMIT = SOCRATA_CATALOGUE_LIMIT;
|
|
8
|
+
/** Committed snapshot, so a build works when the Vermont host is unreachable. */
|
|
9
|
+
const VERMONT_OPEN_DATA_FIXTURE_FILENAME = 'vermont-open-data-catalogue-2026-11-08.json';
|
|
10
|
+
/**
|
|
11
|
+
* Builds the request URL for one page of the Vermont catalogue.
|
|
12
|
+
*
|
|
13
|
+
* @param options - an optional entry cap
|
|
14
|
+
* @returns the full request URL
|
|
15
|
+
*/
|
|
16
|
+
export function buildVermontOpenDataCatalogueUrl(options) {
|
|
17
|
+
return buildSocrataCatalogueUrl(VERMONT_OPEN_DATA_API_BASE, options);
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Parses one page of the Vermont catalogue into entries and topic counts.
|
|
21
|
+
*
|
|
22
|
+
* @param payload - the JSON body from the Vermont Socrata discovery API
|
|
23
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
24
|
+
*/
|
|
25
|
+
export function parseVermontOpenDataCataloguePayload(payload) {
|
|
26
|
+
return parseSocrataCataloguePayload(VERMONT_OPEN_DATA_CATALOGUE_SOURCE_ID, payload);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Reads one page of the Vermont Open Data catalogue.
|
|
30
|
+
*
|
|
31
|
+
* @param options - an optional entry cap and fetch stub
|
|
32
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
33
|
+
*/
|
|
34
|
+
export async function fetchVermontOpenDataCatalogue(options) {
|
|
35
|
+
return fetchSocrataCatalogue({ sourceId: VERMONT_OPEN_DATA_CATALOGUE_SOURCE_ID, apiBase: VERMONT_OPEN_DATA_API_BASE }, options);
|
|
36
|
+
}
|
|
37
|
+
/** Vermont Open Data catalogue of datasets and filtered views, keyless. */
|
|
38
|
+
export const vermontOpenDataCatalogueAdapter = createSocrataCatalogueAdapter({
|
|
39
|
+
sourceId: VERMONT_OPEN_DATA_CATALOGUE_SOURCE_ID,
|
|
40
|
+
name: 'Vermont Open Data catalogue',
|
|
41
|
+
description: 'The Vermont Open Data Socrata catalogue, one row per published dataset or filtered view, with its state topic, view count, and download count.',
|
|
42
|
+
apiBase: VERMONT_OPEN_DATA_API_BASE,
|
|
43
|
+
fixtureFilename: VERMONT_OPEN_DATA_FIXTURE_FILENAME,
|
|
44
|
+
});
|
|
45
|
+
//# sourceMappingURL=vermontOpenDataCatalogue.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vermontOpenDataCatalogue.js","sourceRoot":"","sources":["../src/vermontOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,wBAAwB,EACxB,6BAA6B,EAC7B,qBAAqB,EACrB,4BAA4B,EAC5B,uBAAuB,GAIxB,MAAM,uBAAuB,CAAC;AAE/B,6DAA6D;AAC7D,MAAM,CAAC,MAAM,qCAAqC,GAAG,6BAA6B,CAAC;AAEnF,4DAA4D;AAC5D,MAAM,CAAC,MAAM,0BAA0B,GAAG,yCAAyC,CAAC;AAEpF,oDAAoD;AACpD,MAAM,CAAC,MAAM,iCAAiC,GAAG,uBAAuB,CAAC;AAWzE,iFAAiF;AACjF,MAAM,kCAAkC,GAAG,6CAA6C,CAAC;AAEzF;;;;;GAKG;AACH,MAAM,UAAU,gCAAgC,CAAC,OAA4B;IAC3E,OAAO,wBAAwB,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;AACvE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,oCAAoC,CAAC,OAAgB;IACnE,OAAO,4BAA4B,CAAC,qCAAqC,EAAE,OAAO,CAAC,CAAC;AACtF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CAAC,OAGnD;IACC,OAAO,qBAAqB,CAC1B,EAAE,QAAQ,EAAE,qCAAqC,EAAE,OAAO,EAAE,0BAA0B,EAAE,EACxF,OAAO,CACR,CAAC;AACJ,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,MAAM,+BAA+B,GAAG,6BAA6B,CAAC;IAC3E,QAAQ,EAAE,qCAAqC;IAC/C,IAAI,EAAE,6BAA6B;IACnC,WAAW,EACT,gJAAgJ;IAClJ,OAAO,EAAE,0BAA0B;IACnC,eAAe,EAAE,kCAAkC;CACpD,CAAC,CAAC"}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type SocrataCatalogue, type SocrataCatalogueCategoryCount, type SocrataCatalogueEntry } from './socrataCatalogue.js';
|
|
2
|
+
/** Adapter id for the Washington State Open Data catalogue source. */
|
|
3
|
+
export declare const WASHINGTON_STATE_OPEN_DATA_CATALOGUE_SOURCE_ID = "washington-state-open-data-catalogue";
|
|
4
|
+
/** Base URL for the Washington State Open Data Socrata catalogue. */
|
|
5
|
+
export declare const WASHINGTON_STATE_OPEN_DATA_API_BASE = "https://data.wa.gov/api/views.json";
|
|
6
|
+
/** How many catalogue entries one request reads. */
|
|
7
|
+
export declare const WASHINGTON_STATE_OPEN_DATA_CATALOGUE_LIMIT = 10;
|
|
8
|
+
/** One catalogue entry from the Washington State Socrata discovery API. */
|
|
9
|
+
export type WashingtonStateOpenDataCatalogueEntry = SocrataCatalogueEntry;
|
|
10
|
+
/** One state topic and how many catalogue entries carry it. */
|
|
11
|
+
export type WashingtonStateOpenDataCategoryCount = SocrataCatalogueCategoryCount;
|
|
12
|
+
/** One page of the Washington State catalogue, folded for a story. */
|
|
13
|
+
export type WashingtonStateOpenDataCatalogue = SocrataCatalogue;
|
|
14
|
+
/**
|
|
15
|
+
* Builds the request URL for one page of the Washington State catalogue.
|
|
16
|
+
*
|
|
17
|
+
* @param options - an optional entry cap
|
|
18
|
+
* @returns the full request URL
|
|
19
|
+
*/
|
|
20
|
+
export declare function buildWashingtonStateOpenDataCatalogueUrl(options?: {
|
|
21
|
+
limit?: number;
|
|
22
|
+
}): string;
|
|
23
|
+
/**
|
|
24
|
+
* Parses one page of the Washington State catalogue into entries and topic counts.
|
|
25
|
+
*
|
|
26
|
+
* @param payload - the JSON body from the Washington State Socrata discovery API
|
|
27
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
28
|
+
*/
|
|
29
|
+
export declare function parseWashingtonStateOpenDataCataloguePayload(payload: unknown): WashingtonStateOpenDataCatalogue;
|
|
30
|
+
/**
|
|
31
|
+
* Reads one page of the Washington State Open Data catalogue.
|
|
32
|
+
*
|
|
33
|
+
* @param options - an optional entry cap and fetch stub
|
|
34
|
+
* @returns the entries in the page, the counts, and the newest entry
|
|
35
|
+
*/
|
|
36
|
+
export declare function fetchWashingtonStateOpenDataCatalogue(options?: {
|
|
37
|
+
limit?: number;
|
|
38
|
+
fetchImpl?: typeof globalThis.fetch;
|
|
39
|
+
}): Promise<WashingtonStateOpenDataCatalogue>;
|
|
40
|
+
/** Washington State Open Data catalogue of datasets, charts, and stories, keyless. */
|
|
41
|
+
export declare const washingtonStateOpenDataCatalogueAdapter: import("./types.js").UsDataAdapter<SocrataCatalogue>;
|
|
42
|
+
//# sourceMappingURL=washingtonStateOpenDataCatalogue.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"washingtonStateOpenDataCatalogue.d.ts","sourceRoot":"","sources":["../src/washingtonStateOpenDataCatalogue.ts"],"names":[],"mappings":"AAAA,OAAO,EAML,KAAK,gBAAgB,EACrB,KAAK,6BAA6B,EAClC,KAAK,qBAAqB,EAC3B,MAAM,uBAAuB,CAAC;AAE/B,sEAAsE;AACtE,eAAO,MAAM,8CAA8C,yCACnB,CAAC;AAEzC,qEAAqE;AACrE,eAAO,MAAM,mCAAmC,uCAAuC,CAAC;AAExF,oDAAoD;AACpD,eAAO,MAAM,0CAA0C,KAA0B,CAAC;AAElF,2EAA2E;AAC3E,MAAM,MAAM,qCAAqC,GAAG,qBAAqB,CAAC;AAE1E,+DAA+D;AAC/D,MAAM,MAAM,oCAAoC,GAAG,6BAA6B,CAAC;AAEjF,sEAAsE;AACtE,MAAM,MAAM,gCAAgC,GAAG,gBAAgB,CAAC;AAMhE;;;;;GAKG;AACH,wBAAgB,wCAAwC,CAAC,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,MAAM,CAE7F;AAED;;;;;GAKG;AACH,wBAAgB,4CAA4C,CAC1D,OAAO,EAAE,OAAO,GACf,gCAAgC,CAElC;AAED;;;;;GAKG;AACH,wBAAsB,qCAAqC,CAAC,OAAO,CAAC,EAAE;IACpE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,OAAO,UAAU,CAAC,KAAK,CAAC;CACrC,GAAG,OAAO,CAAC,gCAAgC,CAAC,CAQ5C;AAED,sFAAsF;AACtF,eAAO,MAAM,uCAAuC,sDAOlD,CAAC"}
|