@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/dk/tools/fetch/inspire.ts

@@ -0,0 +1,323 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Fetch Denmark's INSPIRE Addresses GeoPackage for the `dk-inspire` adapter.
+ *
+ * Denmark is one of the member states that serves the Addresses (AD) theme as a predefined-dataset
+ * ATOM feed rather than as a WFS, so `#tools/fetch/inspire-addresses` — which pages a WFS and joins
+ * its feature types locally — does not reach it. This module reads the feed, downloads the one
+ * dataset it offers and extracts the GeoPackage.
+ *
+ * The acquisition belongs under a `tools/` root. `@mailwoman/corpus` is one of the
+ * `TOOLING_PACKAGES` in `dependency-cruiser.config.mjs`, which keep their tooling under `lib/`, so
+ * this module sits beside `#fr/tools/fetch/ban` rather than in an `sdk/` root the workspace does
+ * not declare.
+ *
+ * Klimadatastyrelsen states CC BY 4.0 for this theme, and the address-source register elects it. The
+ * model card must attribute Klimadatastyrelsen.
+ *
+ * Five properties of this host were measured, and each one breaks a reader that assumes otherwise:
+ *
+ * 1. `content-length` on the feed is the gzip-compressed length. The header reports 1,455 while 3,130
+ *    bytes are delivered, so a reader that trusts it truncates the feed. Every byte count here is
+ *    counted off the delivered body.
+ * 2. The entry's `rel="alternate"` href is a Wing FTP weblink landing page, HTTP 200 `text/html`,
+ *    3,811 bytes. The file needs `&realfilename=DK_INSPIRE_Addresses.zip` appended. That query answers
+ *    HTTP 200 `application/zip`.
+ * 3. The host ignores `Range`. `bytes=0-4095` answers HTTP 200 with no `content-range` and the whole
+ *    101 MB body, so the one-byte range GET that sizes an OpenAddresses source cannot be used here.
+ * 4. The entry's own `length="4291699"` understates the real 101,406,038 bytes by 23 times. The
+ *    download is never sized from the feed.
+ * 5. The feed's `<updated>` is 2020-12-24 and the entry's 2022-10-06, both stale against the file's
+ *    own 2026-09-27 modification date, so a freshness check cannot read the feed's timestamps. This
+ *    module compares the downloaded zip's sha256 against its manifest instead.
+ *
+ * The linked OpenSearch description parameterises only `spatial_dataset_identifier_code`, `crs` and
+ * `language`, and offers no spatial subsetting, so there is no way to fetch part of Denmark. The
+ * feed holds one entry covering the whole country.
+ */
+
+/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
+   procedure rather than sections of declarations. A region there folds no element a reader wants folded. */
+
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { pathExists, statPath } from "@mailwoman/core/fs/readers/stat"
+import { makeDirectories, removePathIfPresent } from "@mailwoman/core/fs/writers"
+import { extractZipEntry, listZipEntries } from "@mailwoman/core/fs/zip"
+import { sha256File } from "@mailwoman/core/hash"
+import { childElement, childElements, type MarkupElement, streamMarkupElements } from "@mailwoman/core/html/elements"
+
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { readManifest, streamBodyToFile, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The ATOM service document for Denmark's Addresses theme.
+ */
+export const DK_ADDRESSES_FEED_URL = "https://www2.sdfe.dk/INSPIRE/feeds/Addresses.xml"
+
+/**
+ * The adapter slug this module fetches for, and the directory it writes under.
+ */
+const SLUG = "dk-inspire"
+
+/**
+ * The zip member the adapter reads.
+ */
+export const DK_ADDRESSES_GEOPACKAGE_MEMBER = "ad_inspire.gpkg"
+
+/**
+ * The query parameter that turns the entry's weblink landing page into the file itself.
+ *
+ * Wing FTP serves the file only when the request asks for the member.
+ */
+const REAL_FILENAME_PARAMETER = "realfilename"
+
+/**
+ * The smallest plausible zip.
+ * Anything smaller is an error page rather than the dataset.
+ *
+ * The real file is 101,406,038 bytes, and this floor only separates a body from a page.
+ */
+const MINIMUM_DATASET_BYTES = 1_000_000
+
+/**
+ * What the feed states about its one dataset.
+ */
+export interface AddressesFeedEntry {
+	/**
+	 * The entry's title, `DK INSPIRE Addresses` in the measured feed.
+	 */
+	title: string
+	/**
+	 * The download URL, with the member in the query so the host serves the file.
+	 */
+	downloadURL: string
+	/**
+	 * The media type the entry advertises, `application/geopackage+sqlite3` in the measured feed.
+	 */
+	mediaType: string | null
+}
+
+/**
+ * Read the one dataset entry out of the feed.
+ *
+ * Parsed with `@mailwoman/core/html/elements` rather than matched with a pattern.
+ *
+ * The feed needs both of the things a pattern handles badly.
+ * Its entry tag has an `xml:lang` attribute, so a reader has to keep attributes.
+ *
+ * Its hrefs escape the ampersand between query parameters, so a reader has to decode entities.
+ *
+ * The parser does both.
+ * XML mode is required: the entry's links self-close, and HTML mode leaves them open.
+ *
+ * @param xml The whole feed, 3,130 bytes, taken as one string rather than as a stream.
+ * @throws When the feed holds no entry, or its entry no usable link.
+ * A feed holding neither reads as a change at the publisher.
+ * A caller has to see that.
+ */
+export async function readAddressesFeed(xml: string, feedURL: string): Promise<AddressesFeedEntry> {
+	async function* oneChunk(): AsyncIterable<string> {
+		yield xml
+	}
+
+	// The feed holds one entry covering the whole country, so the first is the dataset.
+	let entry: MarkupElement | null = null
+
+	for await (const found of streamMarkupElements(oneChunk(), "entry", { xml: true })) {
+		entry ??= found
+	}
+
+	if (!entry) {
+		throw new Error(`${feedURL}: the feed holds no <entry>, so the dataset it offers could not be read.`)
+	}
+
+	const title = childElement(entry, "title")?.text ?? ""
+
+	// The alternate link is the dataset.
+	// The same entry also has a `describedby` link to the ISO 19139 record.
+	// Either would download as a dataset without this check.
+	const alternate = childElements(entry, "link").find(
+		(link: MarkupElement) => (link.attributes.rel ?? "").toLowerCase() === "alternate"
+	)
+
+	if (!alternate) {
+		throw new Error(
+			`${feedURL}: the entry titled "${title}" carries no rel="alternate" link, so its dataset URL could not be read.`
+		)
+	}
+
+	const href = alternate.attributes.href
+
+	if (!href) {
+		throw new Error(`${feedURL}: the entry titled "${title}" carries an alternate link with no href.`)
+	}
+
+	return {
+		title,
+		// The parser decodes `&amp;` in an attribute value, so the href arrives with
+		// its query parameters already separated by a bare `&`.
+		downloadURL: datasetURLOf(href),
+		mediaType: alternate.attributes.type ?? null,
+	}
+}
+
+/**
+ * Name the zip member on a weblink URL, so the host serves the file instead of its landing page.
+ *
+ * A URL that already names one is answered unchanged, so a publisher that starts writing
+ * the complete URL into the feed does not get the parameter twice.
+ */
+export function datasetURLOf(href: string): string {
+	if (href.includes(`${REAL_FILENAME_PARAMETER}=`)) return href
+
+	return `${href}&${REAL_FILENAME_PARAMETER}=${encodeURIComponent("DK_INSPIRE_Addresses.zip")}`
+}
+
+export type FetchDKAddressesOptions = BaseFetchOptions
+
+/**
+ * Download Denmark's Addresses GeoPackage and leave it beside a manifest recording its sha256.
+ *
+ * Re-runnable: a run whose manifest sha256 still matches the extracted GeoPackage
+ * reuses that file and makes no request.
+ */
+export async function fetchDKAddresses(
+	options: FetchDKAddressesOptions,
+	report?: (line: string) => void
+): Promise<FetchSummary> {
+	const destDir = options.outRoot(SLUG)
+
+	await makeDirectories(destDir)
+
+	const gpkgDest = destDir(DK_ADDRESSES_GEOPACKAGE_MEMBER)
+	const zipDest = destDir("DK_INSPIRE_Addresses.zip")
+	const manifestPath = destDir("MANIFEST.json")
+
+	report?.(`=== ${SLUG}`)
+
+	// MARK: Idempotency check
+	//
+	// The feed's own timestamps are older than the file it serves, so they cannot decide this.
+	// The extracted GeoPackage's digest can.
+
+	const recorded = await readManifest<Partial<SourceManifest>>(manifestPath)
+
+	if (recorded?.sha256 && recorded.filename) {
+		const recordedPath = destDir(recorded.filename)
+
+		if ((await pathExists(recordedPath)) && (await sha256File(recordedPath)) === recorded.sha256) {
+			report?.("  ✓ Already current (sha256 matches MANIFEST) — skipping download.")
+
+			return { fetched: 0, skipped: 1, failed: 0, failedCodes: [] }
+		}
+	}
+
+	// MARK: Read the feed
+
+	report?.(`  Reading ${DK_ADDRESSES_FEED_URL} ...`)
+
+	const feedResponse = await fetch(DK_ADDRESSES_FEED_URL, { redirect: "follow" })
+
+	if (!feedResponse.ok) {
+		report?.(`  ✗ HTTP ${feedResponse.status} reading the feed`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// `text()` reads the whole decoded body.
+	// The `content-length` header on this host reports the compressed length,
+	// so it is neither read nor checked against what arrived.
+	const xml = await feedResponse.text()
+
+	// Characters rather than bytes: the feed's 3,130 bytes decode to 3,125 characters,
+	// because the Danish text in it is multi-byte.
+	// Its content-length header reports the compressed length, 1,455, rather than either of those.
+	report?.(`  Feed decoded to ${xml.length} characters`)
+
+	const entry = await readAddressesFeed(xml, DK_ADDRESSES_FEED_URL)
+
+	report?.(`  Entry: ${entry.title} (${entry.mediaType ?? "no advertised type"})`)
+	report?.(`  Dataset: ${entry.downloadURL}`)
+
+	// MARK: Download the dataset
+	//
+	// Streamed rather than buffered, and the byte count comes from the body
+	// rather than from the entry's `length` attribute.
+	// That attribute understates the file by 23 times.
+
+	const datasetResponse = await fetch(entry.downloadURL, { redirect: "follow" })
+
+	if (!datasetResponse.ok) {
+		report?.(`  ✗ HTTP ${datasetResponse.status} downloading the dataset`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const contentType = datasetResponse.headers.get("content-type") ?? ""
+
+	// The bare weblink href answers HTTP 200 `text/html` with a landing page.
+	// A write to disk would fail later as a corrupt zip.
+	if (contentType.includes("text/html")) {
+		report?.(`  ✗ The dataset URL answered ${contentType}, which is the weblink's landing page rather than the file`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	const zipBytes = await streamBodyToFile(datasetResponse, zipDest)
+
+	report?.(`  Downloaded: ${ByteFormatter.formatIEC(zipBytes)} (${zipBytes} bytes)`)
+
+	if (zipBytes < MINIMUM_DATASET_BYTES) {
+		report?.(`  ✗ Response too small (${zipBytes} bytes) — probable error page`)
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	// MARK: Extract the GeoPackage
+
+	const entries = await listZipEntries(zipDest)
+	const member = entries.find((zipEntry) => zipEntry.name.endsWith(".gpkg"))
+
+	if (!member) {
+		report?.("  Members in the archive:")
+
+		for (const zipEntry of entries) {
+			report?.(`    ${zipEntry.name}`)
+		}
+
+		report?.("  ✗ The archive holds no .gpkg member — inspect the listing above and update this module")
+
+		return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] }
+	}
+
+	report?.(`  Extracting ${member.name} (${ByteFormatter.formatIEC(member.uncompressedSize)}) ...`)
+
+	await extractZipEntry(zipDest, member.name, gpkgDest)
+
+	const gpkgSize = (await statPath(gpkgDest)).size
+	const gpkgSha = await sha256File(gpkgDest)
+
+	await removePathIfPresent(zipDest)
+
+	report?.("  Removed the archive (the GeoPackage is kept)")
+
+	// MARK: Write the manifest
+
+	const manifest: SourceManifest = {
+		source_url: entry.downloadURL,
+		downloaded_at: new Date().toISOString(),
+		filename: DK_ADDRESSES_GEOPACKAGE_MEMBER,
+		sha256: gpkgSha,
+		bytes: gpkgSize,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	report?.(`  ✓ ${ByteFormatter.formatIEC(gpkgSize)}  sha256=${gpkgSha}`)
+	report?.(`  MANIFEST written to ${manifestPath}`)
+
+	return { fetched: 1, skipped: 0, failed: 0, failedCodes: [] }
+}

package/lib/ee/adapters/ads/adapter.ts

@@ -0,0 +1,240 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `ads`: Estonia's INSPIRE Addresses (AD) theme, published by Maa- ja Ruumiamet out of the
+ * Aadressiandmete süsteem.
+ *
+ * Input is newline-delimited GeoJSON: one `AD_Address:AD.Address` feature per line, as the service's
+ * `outputFormat=application/json` writes them. A page of the service is a `FeatureCollection`, and the
+ * acquisition step appends each page's `features` to the file, so the adapter streams a 729,973-feature
+ * extract without holding a page in memory.
+ *
+ * GeoServer flattens the INSPIRE application schema here, so every component value arrives inline on
+ * the feature's `properties` and the adapter resolves no reference. The county, the municipality, the
+ * settlement unit, the address area, the thoroughfare and the postal descriptor each occupy one of six
+ * fixed `component<n>_xlink_title` slots. An unused slot holds the string `unpopulated` rather than a
+ * void object. The slot a value lands in is fixed: slot 1 holds the county, slot 2 the municipality,
+ * slot 3 the settlement unit, slot 4 the address area, slot 5 the thoroughfare and slot 6 the postal
+ * descriptor.
+ *
+ * The field whose name matches the INSPIRE designator holds an internal ADS code, and the house
+ * number sits in the locator's name-spelling element. Estonia writes that ADS code in
+ * `locator_addresslocator_designator_locatordesignator_designator` (`7_1DOW`, `7_3P4Y`, `6_0MU3`).
+ * The human-readable number is in `locator_addresslocator_name_locatorname_name_spelling_text`. The
+ * designator's own type states `addressNumber` on the street rows, so that field is tempting to
+ * read. A reader that takes it would stamp `7_1DOW` as the house number on every row.
+ *
+ * The locator name takes one of two roles, and the feature's structure decides which. A feature that
+ * holds a thoroughfare or an address area writes its number there, and one that holds neither writes
+ * the farm or building name that designates the addressable unit. The two never co-occur. The adapter
+ * therefore reads `house_number` where a thoroughfare element exists and `venue` where none does, so a
+ * farm name never arrives labeled as a number.
+ *
+ * Slots 4 and 5 hold a malformed `href`: it addresses the `AU_haldusyksused` workspace with an
+ * `AD_Address` type and answers HTTP 400 `Unknown namespace [AD_Address]`. The adapter reads the slot's
+ * `title` and never follows an `href`.
+ *
+ * A row with no thoroughfare element is the ordinary rural case rather than a defect.
+ *
+ * Maa- ja Ruumiamet's dataset record states Creative Commons CC0 1.0 in `otherConstraints`. The
+ * address-source register elected that license and records it under `spdx` as `CC0-1.0`. CC0 reserves no act, so the
+ * adapter stamps that label on every row and the model card needs no attribution clause, although
+ * crediting the publisher stays good practice.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `EE`.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { JSONSpliterator } from "spliterator"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const ADS_ADAPTER_ID = "ads"
+
+/**
+ * The one jurisdiction this adapter emits.
+ *
+ * The service covers Estonia whole, so the set has one member and `opts.country` either
+ * matches `EE` or asks for a jurisdiction the dataset does not publish.
+ */
+export const ADS_COUNTRIES: readonly string[] = ["EE"]
+
+/**
+ * The license the address-source register elected for this publisher.
+ *
+ * `unchecked-access-free-ee-maa-ja-ruumiamet` is elected with `spdx` `CC0-1.0`,
+ * from the `otherConstraints` of CSW record `b0bd266e-b4c6-45db-b7ea-f47486b6258b`.
+ */
+export const ADS_DEFAULT_LICENSE = "CC0-1.0"
+
+/**
+ * The sentinel Estonia writes in an unused component slot.
+ *
+ * It is the local name of an INSPIRE void reason, written as bare text where GeoServer would
+ * otherwise write a void object, and it appears in both the slot's `href` and its `title`.
+ * A reader that takes the title at face value stores the word as a street name.
+ */
+const UNPOPULATED = "unpopulated"
+
+/**
+ * The flattened `AD.Address` properties this adapter consults.
+ *
+ * The explicit shape catches a rename early, and leaving the designator out of
+ * it is deliberate: the property exists on every feature and holds an ADS code
+ * rather than a house number, so the adapter cannot read it by accident.
+ */
+interface ADSProperties {
+	gml_id?: number | string
+	inspireid_identifier_localid?: number | string
+	locator_addresslocator_name_locatorname_name_spelling_text?: string
+	component1_xlink_title?: string
+	component2_xlink_title?: string
+	component3_xlink_title?: string
+	component4_xlink_title?: string
+	component5_xlink_title?: string
+	component6_xlink_title?: string
+}
+
+/**
+ * One line of the input: a GeoJSON feature as the service's JSON writer emits it.
+ */
+interface ADSFeature {
+	id?: string
+	properties?: ADSProperties
+}
+
+/**
+ * A component slot's value, or an empty string where the slot is unused.
+ *
+ * The sentinel is compared case-insensitively because it renders an INSPIRE codelist term
+ * rather than a value the publisher authored.
+ */
+function slotValue(value: string | null | undefined): string {
+	const trimmed = (value ?? "").trim()
+
+	return trimmed.toLowerCase() === UNPOPULATED ? "" : trimmed
+}
+
+/**
+ * The thoroughfare element a feature addresses, or an empty string where it addresses none.
+ *
+ * A thoroughfare name and an address area never co-occur on one feature, and the
+ * publisher's own `alternativeidentifier` writes either one in the same position:
+ * `Veskimäe tee 8` for the first and `Vahatu vkt 19` for the second.
+ * The address area is therefore read as the street it stands in for rather than as a separate rung.
+ */
+export function thoroughfareOf(properties: ADSProperties): string {
+	return slotValue(properties.component5_xlink_title) || slotValue(properties.component4_xlink_title)
+}
+
+export function createADSAdapter(): CorpusAdapter {
+	return {
+		id: ADS_ADAPTER_ID,
+		defaultLicense: ADS_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.EstoniaADS,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"Estonia's INSPIRE Addresses theme (Maa- ja Ruumiamet): address points with a thoroughfare or a named site.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !ADS_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(ADS_ADAPTER_ID, ADS_COUNTRIES, opts.country)
+			}
+
+			const features = JSONSpliterator.fromAsync<ADSFeature>(opts.inputPath)
+
+			let emitted = 0
+
+			for await (const feature of features) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const properties = feature.properties ?? {}
+
+				const region = slotValue(properties.component1_xlink_title)
+				const municipality = slotValue(properties.component2_xlink_title)
+				const settlement = slotValue(properties.component3_xlink_title)
+				const street = thoroughfareOf(properties)
+				const postcode = slotValue(properties.component6_xlink_title)
+				const locatorName = (properties.locator_addresslocator_name_locatorname_name_spelling_text ?? "").trim()
+
+				if (!municipality) continue
+
+				if (!postcode && !settlement) continue
+
+				const components: CanonicalRow["components"] = {}
+
+				if (street) {
+					// A thoroughfare or an address area makes the locator name this feature's
+					// number on that thoroughfare.
+					if (locatorName) {
+						components.house_number = locatorName
+					}
+
+					components.street = street
+				} else if (locatorName) {
+					// With no thoroughfare the locator name is the site or building name
+					// that designates the addressable unit.
+					// That is how a rural Estonian address works.
+					components.venue = locatorName
+				}
+
+				// The settlement unit sits inside the municipality, and Estonia writes it between
+				// the thoroughfare and the municipality in its own `alternativeidentifier`.
+				if (settlement) {
+					components.dependent_locality = settlement
+				}
+
+				if (postcode) {
+					components.postcode = postcode
+				}
+
+				components.locality = municipality
+
+				if (region) {
+					components.region = region
+				}
+
+				const rendered = formatAddressRow(components, "EE", { singleLine: true })
+
+				if (!rendered) continue
+
+				const { raw, components: aligned } = rendered
+
+				const seed = String(properties.inspireid_identifier_localid ?? properties.gml_id ?? feature.id ?? "").trim()
+
+				const sourceID = seed ? `${ADS_ADAPTER_ID}-${seed}` : stableSourceID(ADS_ADAPTER_ID, aligned)
+
+				yield {
+					raw,
+					components: aligned,
+					country: "EE",
+					locale: "et-EE",
+					source: ADS_ADAPTER_ID,
+					source_id: sourceID,
+					corpus_version: "",
+					license: ADS_DEFAULT_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const adsAdapter = createADSAdapter()