@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/lv/tools/fetch/varis.ts

@@ -0,0 +1,215 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Downloads the State Address Register's open data (`Valsts adrešu reģistra atvērtie dati`) for the
+ * `lv-varis` adapter.
+ *
+ * Valsts zemes dienests republishes every table of the dataset `varis-atvertie-dati` on data.gov.lv,
+ * and a republish can move a resource's URL. The fetcher therefore asks the portal's CKAN API for the
+ * dataset's resources and picks each table by the file name its URL ends in, rather than holding URLs
+ * that the next republish may retire. The manifest records, per file, the resource URL, the CKAN
+ * `last_modified`, the byte count and the SHA-256, and once for the dataset the license id the portal
+ * states, so a corpus row traces to one publication.
+ */
+
+import { APIClient } from "@mailwoman/core/api"
+import { ByteFormatter } from "@mailwoman/core/fs/formatters"
+import { tryStat } from "@mailwoman/core/fs/readers/stat"
+import { makeDirectories } from "@mailwoman/core/fs/writers"
+import { sha256File } from "@mailwoman/core/hash"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { LV_VARIS_ADAPTER_ID, LV_VARIS_LICENSE, VARIS_TABLES, type VarisTable } from "#lv/adapters/varis/adapter"
+import type { CKANResource } from "#ro/tools/fetch/retea-scolara"
+import type { BaseFetchOptions, FetchSummary, SourceManifest } from "#tools/fetch/download"
+import { downloadToFile, readManifest, writeManifest } from "#tools/fetch/download"
+
+/**
+ * The data.gov.lv dataset that holds the register's tables.
+ */
+export const LV_VARIS_DATASET_ID = "varis-atvertie-dati"
+
+/**
+ * The CKAN call that lists the dataset's resources and its license.
+ */
+export const LV_VARIS_PACKAGE_URL = `https://data.gov.lv/dati/api/3/action/package_show?id=${LV_VARIS_DATASET_ID}`
+
+const SLUG = LV_VARIS_ADAPTER_ID
+
+interface CKANPackage {
+	license_id?: string | null
+	license_title?: string | null
+	organization?: { title?: string | null } | null
+	resources?: CKANResource[]
+}
+
+/**
+ * One downloaded table, as the manifest records it.
+ */
+export interface VarisFileManifest extends SourceManifest {
+	resource_name: string | null
+	last_modified: string | null
+}
+
+/**
+ * The manifest written beside the tables.
+ */
+export interface VarisManifest {
+	source: string
+	dataset_url: string
+	publisher: string | null
+	license_id: string | null
+	license_title: string | null
+	files: VarisFileManifest[]
+}
+
+/**
+ * The resource whose URL path ends in `/<filename>`.
+ *
+ * The dataset also lists each table's `_his` history file and a `_metadata.json` beside it,
+ * so the match is on the whole final path segment rather than on a prefix.
+ *
+ * @throws When the dataset lists no resource for `filename`.
+ */
+export function resourceForFile(resources: readonly CKANResource[], filename: string): CKANResource {
+	const found = resources.find((resource) => {
+		const path = URL.canParse(resource.url) ? new URL(resource.url).pathname : resource.url
+
+		return path.split("/").at(-1) === filename
+	})
+
+	if (!found) throw new Error(`${SLUG}: dataset ${LV_VARIS_DATASET_ID} lists no resource named ${filename}`)
+
+	return found
+}
+
+export interface DownloadVarisOptions {
+	outputDir: PathBuilderLike
+	/**
+	 * The tables to download.
+	 *
+	 * @defaultValue every table the adapter reads.
+	 */
+	tables?: readonly VarisTable[]
+	force?: boolean
+	retries?: number
+	retryDelayMs?: number
+	signal?: AbortSignal
+	report?: (line: string) => void
+}
+
+export async function downloadVaris(
+	client: Pick<APIClient, "fetch">,
+	options: DownloadVarisOptions
+): Promise<FetchSummary> {
+	const { report } = options
+	const destDir = PathBuilder.from(options.outputDir)
+	const manifestPath = destDir("MANIFEST.json")
+
+	await makeDirectories(destDir)
+	report?.(`=== ${SLUG}`)
+
+	const response = await client.fetch<{ result: CKANPackage }>({ url: LV_VARIS_PACKAGE_URL, signal: options.signal })
+	const pkg = response.data.result
+
+	if (pkg.license_id !== LV_VARIS_LICENSE) {
+		report?.(`  the portal now states license ${pkg.license_id ?? "none"}, not ${LV_VARIS_LICENSE}`)
+	}
+
+	const recorded = await readManifest<VarisManifest>(manifestPath)
+	const recordedFiles = new Map((recorded?.files ?? []).map((entry) => [entry.filename, entry]))
+	const files: VarisFileManifest[] = []
+	const summary: FetchSummary = { fetched: 0, skipped: 0, failed: 0, failedCodes: [] }
+
+	for (const table of options.tables ?? Object.values(VARIS_TABLES)) {
+		if (options.signal?.aborted) break
+
+		const resource = resourceForFile(pkg.resources ?? [], table)
+		const lastModified = resource.last_modified ?? resource.created ?? null
+		const dest = destDir(table)
+		const previous = recordedFiles.get(table)
+		const stat = await tryStat(dest)
+
+		if (
+			!options.force &&
+			previous &&
+			stat &&
+			previous.source_url === resource.url &&
+			previous.last_modified === lastModified &&
+			previous.bytes === stat.size
+		) {
+			report?.(`  ${table} present, and the portal lists the same upload`)
+			files.push(previous)
+
+			summary.skipped++
+
+			continue
+		}
+
+		const { bytes } = await downloadToFile({
+			url: resource.url,
+			dest,
+			retries: options.retries,
+			retryDelayMs: options.retryDelayMs,
+			report,
+		})
+
+		const sha256 = await sha256File(dest)
+
+		report?.(`  ✓ ${table} ${ByteFormatter.formatIEC(bytes)} sha256=${sha256}`)
+
+		files.push({
+			source_url: resource.url,
+			filename: table,
+			bytes,
+			sha256,
+			downloaded_at: new Date().toISOString(),
+			resource_name: resource.name?.trim() ?? null,
+			last_modified: lastModified,
+		})
+
+		summary.fetched++
+	}
+
+	const manifest: VarisManifest = {
+		source: SLUG,
+		dataset_url: LV_VARIS_PACKAGE_URL,
+		publisher: pkg.organization?.title ?? null,
+		license_id: pkg.license_id ?? null,
+		license_title: pkg.license_title ?? null,
+		files,
+	}
+
+	await writeManifest(manifestPath, manifest)
+
+	return summary
+}
+
+/**
+ * The directory the adapter reads under a fetch root.
+ */
+export function varisInputPath(outRoot: BaseFetchOptions["outRoot"]): PathBuilderLike {
+	return outRoot(SLUG)
+}
+
+export interface FetchVarisOptions extends BaseFetchOptions {
+	force?: boolean
+	retries?: number
+	signal?: AbortSignal
+}
+
+export async function fetchVaris(options: FetchVarisOptions, report?: (line: string) => void): Promise<FetchSummary> {
+	await using client = new APIClient({ displayName: SLUG, retry: true })
+
+	// Awaited rather than returned, so the client is disposed after the downloads finish.
+	return await downloadVaris(client, {
+		outputDir: options.outRoot(SLUG),
+		force: options.force,
+		retries: options.retries,
+		retryDelayMs: options.retryDelayMs,
+		signal: options.signal,
+		report,
+	})
+}

package/lib/nl/adapters/kadaster/adapter.ts

@@ -0,0 +1,386 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `nl-kadaster`: the INSPIRE Addresses theme Kadaster publishes, read from its national GML.
+ *
+ * Kadaster serves the whole country in one file. The ATOM service document at
+ * `https://service.pdok.nl/kadaster/ad/atom/adressen_inspire_geharmoniseerd.xml`, 2,247 bytes, holds
+ * one entry, and its `rel="alternate"` link is
+ * `https://service.pdok.nl/kadaster/ad/atom/downloads/addresses.gml.gz`, 819,465,603 bytes. It
+ * inflates to 29,677,448,685 bytes of GML holding 10,066,060 address features. The sibling feed URL
+ * the entry's own `<id>` names answers HTTP 404, so the download link is the only live one.
+ *
+ * The size decides the shape of the reader. The document is never buffered: the adapter opens the
+ * local `.gml.gz`, inflates it as a byte stream and takes one feature subtree at a time. Each feature
+ * an address references contributes a value rather than its subtree, so the index is
+ * bounded by a string per street and per place name rather than by the document.
+ *
+ * Four things separate this publisher from the Czech one the shared primitives were first written
+ * against, and each was measured over a 40,000,000-byte ranged prefix of the download, read in one
+ * streaming pass:
+ *
+ * 1. The wrapper is `gml:FeatureCollection` with `gml:featureMember`, where ČÚZK writes
+ *    `base:SpatialDataSet` with `base:member`.
+ * 2. Every `ad:component` reference is a local `#` fragment naming a `gml:id` in the same document,
+ *    and 0 of them went unresolved.
+ * 3. The postcode is inline. The prefix holds 0 `ad:PostalDescriptor` features, and the postcode
+ *    arrives as a locator designator typed `postalDeliveryIdentifier`. Its element is present on all
+ *    40,000 addresses read in the final pass and empty on 1,769 of them, so an address with no
+ *    postcode is ordinary here and the publisher's own rendering agrees: Kadaster's Locatieserver
+ *    writes `De Paasweide 72g-11, Appingedam` for one of them, with no postcode.
+ * 4. The number is typed `addressNumber`, and two further types hold the rest of it.
+ *
+ * The number is assembled from three designators, and Kadaster's own rendering of the same records
+ * fixes how. BAG's `huis_nlt` is the house number as Kadaster writes it, and the Locatieserver serves
+ * it beside the `nummeraanduiding` id that this theme's `base:localId` repeats. Three records of the
+ * fixture, matched by that id:
+ *
+ * | `gml:id` | `addressNumber` | `addressNumberExtension` | `addressNumber2ndExtension` | `huis_nlt` |
+ * | --- | --- | --- | --- | --- |
+ * | `…address.0003200000140702` | `72` | `g` | `11` | `72g-11` |
+ * | `…address.0003200000134065` | `10` | (empty) | `I` | `10-I` |
+ * | `…address.0003200000134593` | `4` | (empty) | `01` | `4-01` |
+ *
+ * So the extension joins the number with no separator and the second extension follows a hyphen.
+ * 305,542 of 345,309 addresses in the prefix write an `addressNumberExtension` that is present and
+ * empty, spelled `<ad:designator/>`. That states an address with no extension. That condition differs
+ * from void.
+ *
+ * The place name comes from `ad:AddressAreaName`, the BAG *woonplaats*. The prefix holds exactly one
+ * `ad:AdminUnitName` feature and it is the country, `Nederland` at `1stOrder`, so this feed publishes
+ * no municipality name and the woonplaats is the finest place name available. An address references
+ * that country feature, so the adapter indexes the type and reads no place name from it.
+ *
+ * One published defect shapes what the adapter refuses to do. `ad:situatedWithin` on an
+ * `ad:ThoroughfareName` writes its target with a hyphen where every `gml:id` uses a dot:
+ * `#nl-imbag-ad-addressareaname-3386` against `gml:id="nl-imbag-ad-addressareaname.1000"`. Measured
+ * over a 5,000,000-character window, 479 local fragments resolved to no feature, all of that shape,
+ * against 2,622 dot-form ids and 0 hyphen-form ids. The same element on an `ad:AddressAreaName` uses
+ * the dot form and resolves. A reader walking `ThoroughfareName → situatedWithin → AddressAreaName`
+ * would therefore read an empty locality for every street and report no failure. So the adapter reads
+ * the street and the place from the address's own `ad:component` references, and those use the dot
+ * form. It never follows `ad:situatedWithin`. Every reference it does follow raises when it resolves to no
+ * feature, so the hyphen form reaching an `ad:component` would be reported rather than dropped.
+ *
+ * Unlike Czechia, this publisher offers no string to check a whole rendered line against.
+ * `ad:alternativeIdentifier` is void on every address. The Locatieserver's `weergavenaam` is the
+ * nearest equivalent, and `adapter.test.ts` pins the twelve fixture rows against it.
+ *
+ * Kadaster states one license and states it as a URL. Both `rights` elements of the ATOM service
+ * document read `https://creativecommons.org/publicdomain/zero/1.0/deed.nl`, the Dutch deed page for
+ * CC0 1.0 Universal. The address-source register elected it with `spdx` `CC0-1.0`. CC0 reserves no
+ * act, so every row records that identifier and no attribution clause is owed, although crediting
+ * Kadaster stays good practice.
+ *
+ * The adapter honors `opts.limit` and `opts.signal`. `opts.country` is optional and accepts only `NL`.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { gunzipChunks } from "@mailwoman/core/fs/compression"
+import { openReadStream } from "@mailwoman/core/fs/streams"
+import type { MarkupElement } from "@mailwoman/core/html/elements"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import { composeHouseNumber } from "#adapters/street-line"
+import {
+	adminUnitLevel,
+	componentHrefs,
+	componentJoinKey,
+	designator,
+	designatorsByType,
+	placeName,
+	thoroughfareName,
+	voidDesignatorTypes,
+} from "#inspire/address"
+import { UnresolvedComponentReferenceError, VoidDesignatorError } from "#inspire/errors"
+import { streamInspireRows } from "#inspire/stream"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const NL_KADASTER_ADAPTER_ID = "nl-kadaster"
+
+/**
+ * The publication this adapter reads.
+ *
+ * The literal stands in for a `SourceRegister` member until `lib/registers.ts` declares
+ * `KadasterInspireAddresses: "nl-kadaster-inspire-ad"`, which is the id this value spells.
+ */
+export const NL_KADASTER_REGISTER = SourceRegister.KadasterInspireAddresses
+
+/**
+ * The one jurisdiction this adapter emits.
+ *
+ * The download covers the Netherlands in Europe.
+ * The Caribbean parts of the Kingdom keep their own cadastres.
+ *
+ * Those cadastres publish no INSPIRE Addresses theme, so the set has one member.
+ */
+export const NL_KADASTER_COUNTRIES: readonly string[] = ["NL"]
+
+/**
+ * The license the address-source register elected for this publisher.
+ *
+ * `unchecked-access-free-nl-kadaster` is elected with `electedTerms`
+ * `Creative Commons CC0 1.0 Universal Public Domain Dedication` and `spdx` `CC0-1.0`,
+ * read from the two `rights` elements of the ATOM service document.
+ */
+export const NL_KADASTER_DEFAULT_LICENSE = "CC0-1.0"
+
+/**
+ * The INSPIRE feature types the adapter reads from the document.
+ *
+ * `ad:ThoroughfareName` holds the street and `ad:AddressAreaName` the woonplaats.
+ * `ad:AdminUnitName` is indexed because an address references the country feature
+ * and a reference resolving to no feature raises.
+ * No place name is read from it.
+ *
+ * `ad:PostalDescriptor` is deliberately absent.
+ * The publisher writes none, and a reference to one would resolve to no feature
+ * and be reported rather than silently read as an address with no postcode.
+ *
+ * All four are read in one pass, because the document writes the referenced features and the addresses
+ * interleaved over 29,677,448,685 bytes and a pass per type would inflate the download once per type.
+ */
+const COMPONENT_TYPES = ["ad:ThoroughfareName", "ad:AddressAreaName", "ad:AdminUnitName"] as const
+
+/**
+ * What one referenced feature contributes to an address.
+ *
+ * A value rather than the feature's subtree, because the national document holds hundreds of
+ * thousands of them and a subtree holds its geometry, its lifespan and its name metadata.
+ */
+interface ReferencedValue {
+	/**
+	 * The feature's element name.
+	 * That decides where its value goes.
+	 */
+	readonly type: string
+
+	/**
+	 * The street name or place name the feature states, null when the feature states none.
+	 */
+	readonly name: string | null
+
+	/**
+	 * The administrative level an `ad:AdminUnitName` states, such as `1stOrder`.
+	 */
+	readonly level: string | null
+}
+
+/**
+ * The value a referenced feature contributes, read once as the feature arrives.
+ */
+function referencedValue(feature: MarkupElement): ReferencedValue {
+	if (feature.name === "ad:ThoroughfareName") {
+		return { type: feature.name, name: thoroughfareName(feature), level: null }
+	}
+
+	if (feature.name === "ad:AdminUnitName") {
+		return { type: feature.name, name: placeName(feature), level: adminUnitLevel(feature) }
+	}
+
+	return { type: feature.name, name: placeName(feature), level: null }
+}
+
+/**
+ * The house number as Kadaster writes it in BAG's `huis_nlt`.
+ *
+ * The letter joins the number directly and the addition follows a hyphen.
+ * The table in this module's header reads that off three records matched against
+ * the Locatieserver by their `nummeraanduiding` id.
+ *
+ * @returns The joined number, or null when the publisher states no `addressNumber`.
+ */
+export function dutchHouseNumber(byType: ReadonlyMap<string, readonly string[]>): string | null {
+	const number = designator(byType, "addressNumber")
+
+	if (!number) return null
+
+	const letter = designator(byType, "addressNumberExtension") ?? ""
+	const addition = designator(byType, "addressNumber2ndExtension") ?? ""
+
+	return composeHouseNumber(composeHouseNumber(number, letter), addition, "-")
+}
+
+export function createNLKadasterAdapter(): CorpusAdapter {
+	return {
+		id: NL_KADASTER_ADAPTER_ID,
+		defaultLicense: NL_KADASTER_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: NL_KADASTER_REGISTER,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"INSPIRE Addresses (Kadaster): house-number-level addresses for the Netherlands, one national gzipped GML.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !NL_KADASTER_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(NL_KADASTER_ADAPTER_ID, NL_KADASTER_COUNTRIES, opts.country)
+			}
+
+			let emitted = 0
+
+			for (const documentPath of await documentPaths(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				// The published document writes 300,318 referenced features ahead of its first address
+				// and held 0 of 345,309 measured, so the driver's holding list stays empty here.
+				// It exists because only the document's own ordering separates a reference
+				// that has not arrived yet from one the publisher never wrote.
+				for await (const row of streamInspireRows<ReferencedValue>({
+					chunks: documentChunks(documentPath),
+					componentElements: COMPONENT_TYPES,
+					// One string per feature rather than its subtree.
+					index: referencedValue,
+					compose: (address, referenced, final) => composeRow(address, referenced, { final }),
+					limit: opts.limit === undefined ? undefined : opts.limit - emitted,
+					signal: opts.signal,
+				})) {
+					yield row
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * One address's row, `null` when it holds too little to render, or `"deferred"`
+ * when a reference it states is not indexed yet.
+ *
+ * With `final` set the document has ended, so an unresolved reference is a value the reader
+ * asked for and could not read, and it raises rather than rendering the address without it.
+ */
+function composeRow(
+	address: MarkupElement,
+	referenced: ReadonlyMap<string, ReferencedValue>,
+	options: { final?: boolean } = {}
+): CanonicalRow | null | "deferred" {
+	const addressID = address.attributes["gml:id"]?.trim() ?? null
+	const voided = voidDesignatorTypes(address)
+
+	for (const type of ["addressNumber", "postalDeliveryIdentifier"]) {
+		if (voided.has(type)) throw new VoidDesignatorError(NL_KADASTER_ADAPTER_ID, addressID, type)
+	}
+
+	let street: string | null = null
+	let locality: string | null = null
+
+	for (const href of componentHrefs(address)) {
+		const key = componentJoinKey(href)
+		const feature = key ? referenced.get(key) : undefined
+
+		if (!feature) {
+			if (!options.final) return "deferred"
+
+			throw new UnresolvedComponentReferenceError(NL_KADASTER_ADAPTER_ID, addressID, href)
+		}
+
+		if (feature.type === "ad:ThoroughfareName") {
+			street = feature.name
+		} else if (feature.type === "ad:AddressAreaName") {
+			locality = feature.name
+		}
+
+		// An `ad:AdminUnitName` reference resolves and contributes no component.
+		// The one feature of that type in the document is the country, and its `ad:level`
+		// reads `1stOrder` while the reference's position states no level at all.
+	}
+
+	if (!locality) return null
+
+	const byType = designatorsByType(address)
+	const house = dutchHouseNumber(byType)
+	const postcode = designator(byType, "postalDeliveryIdentifier")
+
+	if (!street && !postcode) return null
+
+	const components: CanonicalRow["components"] = {}
+
+	if (house) {
+		components.house_number = house
+	}
+
+	if (street) {
+		components.street = street
+	}
+
+	if (postcode) {
+		components.postcode = postcode
+	}
+
+	components.locality = locality
+
+	const rendered = formatAddressRow(components, "NL", { singleLine: true })
+
+	if (!rendered) return null
+
+	const { raw, components: aligned } = rendered
+
+	return {
+		raw,
+		components: aligned,
+		country: "NL",
+		locale: "nl-NL",
+		source: NL_KADASTER_ADAPTER_ID,
+		source_id: addressID ? `${NL_KADASTER_ADAPTER_ID}-${addressID}` : stableSourceID(NL_KADASTER_ADAPTER_ID, aligned),
+		corpus_version: "",
+		license: NL_KADASTER_DEFAULT_LICENSE,
+	}
+}
+
+/**
+ * The document's bytes, inflated when the path ends in `.gz`.
+ *
+ * The publisher serves `.gml.gz` and no acquisition step inflates it on the way in,
+ * so a caller hands the adapter the download as it arrived.
+ * A plain `.gml` is read as well, and the committed fixture is one.
+ *
+ * The bytes a reviewer reads in the diff are therefore the bytes the test parses.
+ */
+function documentChunks(documentPath: PathBuilderLike): AsyncIterable<Uint8Array | string> {
+	const path = PathBuilder.from(documentPath)
+	const bytes = openReadStream(path)
+
+	return path.basename().toLowerCase().endsWith(".gz") ? gunzipChunks(bytes) : bytes
+}
+
+/**
+ * Every document under `inputPath`, where `inputPath` is one GML file or a directory holding them.
+ *
+ * A directory is read because the download is one national file today and the
+ * service document could list more than one entry tomorrow.
+ * A directory holding no document raises rather than yielding zero rows, so a wrong
+ * path reports itself instead of reading as an empty publisher.
+ */
+async function documentPaths(inputPath: PathBuilderLike): Promise<readonly PathBuilderLike[]> {
+	const path = PathBuilder.from(inputPath)
+	const basename = path.basename().toLowerCase()
+
+	if (basename.endsWith(".gml") || basename.endsWith(".gml.gz") || basename.endsWith(".xml")) return [path]
+
+	const names = await Globerator.from("*.{gml,gml.gz,xml}", { cwd: path.toString(), absolute: false }).toSorted()
+
+	if (!names.length) {
+		throw new Error(`${NL_KADASTER_ADAPTER_ID} adapter: ${path.toString()} holds no GML document`)
+	}
+
+	return names.map((name) => path(name))
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const nlKadasterAdapter = createNLKadasterAdapter()