@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/es/adapters/navarra/adapter.ts

@@ -0,0 +1,289 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `es-navarra`: the INSPIRE Addresses theme the Gobierno de Navarra publishes, read from its zipped
+ * GML partitions.
+ *
+ * The ATOM service at
+ * `https://filescartografia.navarra.es/2_CARTOGRAFIA_TEMATICA/2_7_CATASTRO/2_7_3_INSPIRE_ATOM/2_7_3_3_AD/Addresses_ServiceATOM_Navarra.xml`
+ * is 671,131 bytes and splits the province across 272 entries. Every entry is titled `Address
+ * Navarra`, so a partition cannot be selected by place from the feed alone. The three measured each
+ * turned out to hold one municipality, partition 1 Abáigar, partition 100 Eulate and partition 250
+ * Bera, but the feed states no such mapping and this adapter claims none. It reads one archive or a
+ * directory of them, which is how a caller harvests all 272.
+ *
+ * This publisher is read entirely from its references' titles, and that is what separates it from
+ * every other INSPIRE reader here. Its own file publishes no `AD:ThoroughfareName`,
+ * `AD:PostalDescriptor` or `AD:AdminUnitName` feature at all: partition 1 holds 29 distinct element
+ * names across 115 addresses and none of them is a component feature. Each address instead carries
+ * six `AD:component` references whose `xlink:title` states the value:
+ *
+ * | `xlink:href` | `xlink:title` | read as |
+ * | --- | --- | --- |
+ * | `…&ID=AD_ADMINUNITNAME_PAI_34000000000#…` | `España` | skipped |
+ * | `…&ID=AD_ADMINUNITNAME_COM_34150000000#…` | `Comunidad Foral de Navarra` | skipped |
+ * | `…&ID=AD_ADMINUNITNAME_MUN_34153131001#…` | `Abáigar` | the locality |
+ * | `http://inspire.ec.europa.eu/codelist/AdministrativeHierarchyLevel/5thOrder` | `Abáigar` | the settlement |
+ * | `ThoroughfareName` | `CALLE CALLEJA` | the street |
+ * | `PostalDescriptor` | `31280` | the postcode |
+ *
+ * The counts are exact and regular: 690 references over 115 addresses in partition 1, 1,728 over 288
+ * in partition 100 and 6,324 over 1,054 in partition 250, which is six per address in each, one of
+ * each shape above.
+ *
+ * The two place titles are not the same value. In Abáigar and Eulate the `5thOrder` title repeats the
+ * municipality, and in Bera it states a *concejo* the municipality contains: `Suspela` and `Dornaku`
+ * beside `Bera`. The fifth order is below the municipal fourth, so the title is written as a
+ * dependent locality where it differs from the municipality and left out where it repeats it.
+ *
+ * `Comunidad Foral de Navarra` is the autonomous community's own name rather than the province name a
+ * Spanish address writes, and the publisher's own address layer carries no region column, so no
+ * `region` is written.
+ *
+ * The publisher's own rendering is in a different service, and it agrees. `IDENA` publishes
+ * `IDENA:DIRECC_Txt_Direcciones`, 181,098 features, each with `MUNICIPIO`, `ENTIDAD`, `CODPOSTAL`,
+ * `VIA` and `PORTAL`. Partition 1's `AD.Address.1` composes `CALLE CALLEJA, 4, 31280 Abáigar`, and
+ * that layer's `DIRECC_Txt_Direcciones.1` reads `MUNICIPIO` `Abáigar`, `ENTIDAD` `Abáigar`,
+ * `CODPOSTAL` `31280`, `VIA` `CALLE CALLEJA`, `PORTAL` `4`. The adapter's integration test checks a
+ * sample of rows against that layer, which is this source's publisher-side oracle.
+ *
+ * Two values state an absence rather than a value. A `PostalDescriptor` title of `0` is the publisher
+ * holding no postcode, measured in Bera, and a designator of `S/N` is *sin número*, which
+ * `#es/cadastre` reads for all four Spanish cadastres. A designator may also carry a space-separated
+ * letter, `18 A` and `34 A` in Eulate, which is kept as the publisher wrote it.
+ */
+
+import { type MarkupElement, streamMarkupElements } from "@mailwoman/core/html/elements"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { cadastralRow, spanishHouseNumber } from "#es/cadastre"
+import {
+	codelistValue,
+	componentJoinKey,
+	componentLinks,
+	designatorsByType,
+	voidDesignatorTypes,
+} from "#inspire/address"
+import { inspireGMLChunks } from "#inspire/archive"
+import { VoidDesignatorError } from "#inspire/errors"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const ES_NAVARRA_ADAPTER_ID = "es-navarra"
+
+/**
+ * The jurisdiction this adapter emits.
+ */
+export const ES_NAVARRA_COUNTRIES: readonly string[] = ["ES"]
+
+/**
+ * The license the address-source register elected for this publisher.
+ *
+ * Navarra states one license and states it in its own feed: the feed-level `rights` of
+ * `Addresses_ServiceATOM_Navarra.xml` reads `This layer is published under the terms of the license
+ * Creative Commons Attribution 4.0 International (CC BY 4.0)`, and all 272 of its entries repeat it.
+ * The version is stated, so the register records `CC-BY-4.0` as the SPDX identifier.
+ *
+ * The row carries that identifier rather than the license's title.
+ * `licenseVerdict` resolves an identifier and reads a title as unrecognized,
+ * and a row whose license resolves to no expression has unknown obligations
+ * rather than none, which the build reports separately from a refusal.
+ */
+export const ES_NAVARRA_LICENSE = "CC-BY-4.0"
+
+/**
+ * The member of a partition's archive that holds the GML.
+ *
+ * Each archive ships one member, `AD_Navarra_<n>.gml`, so the extension selects
+ * it without the adapter knowing the partition number.
+ */
+const GML_MEMBER = /\.gml$/iu
+
+/**
+ * The element each address is written as.
+ *
+ * The publisher binds the INSPIRE namespaces to upper-case prefixes.
+ */
+const ADDRESS_ELEMENT = "AD:Address"
+
+/**
+ * The designator type the publisher writes, on every address of the three partitions measured.
+ */
+const HOUSE_NUMBER_TYPE = "addressIdentifierGeneral"
+
+/**
+ * The reference href that states a street name in its title.
+ */
+const THOROUGHFARE_HREF = "thoroughfarename"
+
+/**
+ * The reference href that states a postcode in its title.
+ */
+const POSTAL_DESCRIPTOR_HREF = "postaldescriptor"
+
+/**
+ * The stored-query id prefix of the administrative-unit reference that is the municipality.
+ *
+ * The other two are `AD_ADMINUNITNAME_PAI_`, the country, and `AD_ADMINUNITNAME_COM_`,
+ * the autonomous community, and neither belongs to a written address.
+ */
+const MUNICIPALITY_ID_PREFIX = "AD_ADMINUNITNAME_MUN_"
+
+/**
+ * The administrative level whose title is a settlement within the municipality.
+ */
+const SETTLEMENT_LEVEL = "5thOrder"
+
+/**
+ * The postcode title that states that the publisher holds none.
+ *
+ * Measured in Bera, which writes `0` beside its `31780`.
+ */
+const NO_POSTCODE_TITLE = "0"
+
+/**
+ * What one address's references state, read from their titles.
+ */
+interface ReferencedTitles {
+	street?: string
+	postcode?: string
+	locality?: string
+	settlement?: string
+}
+
+/**
+ * The street, postcode and place names an address's own references state.
+ *
+ * Each reference is placed by its href rather than by its position, because the six are written in one
+ * order in the three partitions measured and that order is the publisher's rather than the schema's.
+ */
+export function navarraReferencedTitles(address: MarkupElement): ReferencedTitles {
+	const titles: ReferencedTitles = {}
+
+	for (const { href, title } of componentLinks(address)) {
+		if (!title) continue
+
+		const lowered = href.toLowerCase()
+
+		if (lowered === THOROUGHFARE_HREF) {
+			titles.street = title
+		} else if (lowered === POSTAL_DESCRIPTOR_HREF) {
+			titles.postcode = title === NO_POSTCODE_TITLE ? undefined : title
+		} else if (codelistValue(href) === SETTLEMENT_LEVEL) {
+			titles.settlement = title
+		} else if (componentJoinKey(href)?.startsWith(MUNICIPALITY_ID_PREFIX)) {
+			titles.locality = title
+		}
+	}
+
+	return titles
+}
+
+export function createESNavarraAdapter(): CorpusAdapter {
+	return {
+		id: ES_NAVARRA_ADAPTER_ID,
+		defaultLicense: ES_NAVARRA_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.NavarraInspireAddresses,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"INSPIRE Addresses (Gobierno de Navarra): house-number-level addresses for Navarra, as 272 zipped GML partitions.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !ES_NAVARRA_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(ES_NAVARRA_ADAPTER_ID, ES_NAVARRA_COUNTRIES, opts.country)
+			}
+
+			let emitted = 0
+
+			for (const archivePath of await archivePaths(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				// The markup stream is read directly rather than through `#inspire/stream`,
+				// because this publisher writes no component feature for an address to reference.
+				// The index would stay empty and no address would ever be deferred.
+				for await (const address of streamMarkupElements(inspireGMLChunks(archivePath, GML_MEMBER), ADDRESS_ELEMENT, {
+					xml: true,
+				})) {
+					if (opts.signal?.aborted) break
+
+					if (opts.limit !== undefined && emitted >= opts.limit) break
+
+					const row = composeRow(address)
+
+					if (!row) continue
+
+					yield row
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * One address's row, or undefined when it carries too little to render.
+ */
+function composeRow(address: MarkupElement): CanonicalRow | undefined {
+	const addressID = address.attributes["gml:id"]?.trim()
+
+	if (voidDesignatorTypes(address).has(HOUSE_NUMBER_TYPE)) {
+		throw new VoidDesignatorError(ES_NAVARRA_ADAPTER_ID, addressID, HOUSE_NUMBER_TYPE)
+	}
+
+	const { street, postcode, locality, settlement } = navarraReferencedTitles(address)
+
+	// Navarra's settlement is a fifth-order title, a *concejo* within the municipality, which
+	// `cadastralRow` writes as a dependent locality only where it differs from the municipality.
+	return cadastralRow(
+		{
+			street,
+			house: spanishHouseNumber(designatorsByType(address), HOUSE_NUMBER_TYPE),
+			postcode,
+			locality,
+			settlement,
+			addressID,
+		},
+		{ adapterID: ES_NAVARRA_ADAPTER_ID, license: ES_NAVARRA_LICENSE }
+	)
+}
+
+/**
+ * Every archive under `inputPath`, which is one `.zip`, one unpacked `.gml`, or a directory holding either.
+ *
+ * A directory is the ordinary case, because the publisher ships 272 partitions
+ * and a caller harvesting the province points at the directory it fetched them into.
+ * A directory holding neither raises rather than yielding zero rows, so a wrong path
+ * reports itself instead of reading as an empty publisher.
+ */
+async function archivePaths(inputPath: PathBuilderLike): Promise<readonly PathBuilderLike[]> {
+	const path = PathBuilder.from(inputPath)
+	const basename = path.basename().toLowerCase()
+
+	if (basename.endsWith(".zip") || basename.endsWith(".gml")) return [path]
+
+	const names = await Globerator.from("*.{zip,gml}", { cwd: path.toString(), absolute: false }).toSorted()
+
+	if (!names.length) {
+		throw new Error(`${ES_NAVARRA_ADAPTER_ID} adapter: ${path.toString()} holds no .zip archive or .gml member`)
+	}
+
+	return names.map((name) => path(name))
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const esNavarraAdapter = createESNavarraAdapter()

package/lib/es/cadastre.ts

@@ -0,0 +1,194 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * The two ways a Spanish cadastre states that it holds no value, which all four of them share.
+ *
+ * Spain publishes the INSPIRE Addresses theme through four cadastres: the Dirección General del
+ * Catastro for 52 provinces, and the Diputación Foral de Bizkaia, the Diputación Foral de Gipuzkoa
+ * and the Gobierno de Navarra for their own territories. Each has its own adapter, because each
+ * states its own terms and writes its own file layout. They agree on these two conventions, and
+ * reading either of them as data rather than as an absence puts a token where a corpus row expects a
+ * number or a postcode.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+
+import { stableSourceID } from "#adapters/source-id"
+import { designator } from "#inspire/address"
+import type { CanonicalRow } from "#types"
+
+/**
+ * The *sin número* tokens the Spanish cadastres write, lower-cased for comparison.
+ *
+ * `S-N` is the Dirección General del Catastro's spelling, on 856 of Ceuta's 8,126 addresses.
+ * `S/N` is the Gobierno de Navarra's and the Diputación Foral de Gipuzkoa's.
+ *
+ * Both are accepted by each of the four readers.
+ * Neither is a number under any jurisdiction's rules, and the files measured are a few municipalities
+ * out of the 52 provinces, 112 municipalities and 272 partitions those services publish,
+ * so a reader scoped to the one spelling its own sample carried would write the other into a row.
+ */
+const SIN_NUMERO = new Set(["s-n", "s/n"])
+
+/**
+ * Whether a designator value states that the address has no number.
+ *
+ * The whole value has to be the token.
+ * A value that carries more, such as Gipuzkoa's `S/N-A`, `S/N-C` and `S/N-5`,
+ * states something further and is kept as the publisher wrote it: what the trailing part
+ * designates is not established, dropping it would merge several addresses of one building,
+ * and rewriting it would state a number the publisher did not.
+ */
+export function isSinNumero(value: string): boolean {
+	return SIN_NUMERO.has(value.trim().toLowerCase())
+}
+
+/**
+ * The house number a Spanish cadastre states, or undefined where it states that there is none.
+ *
+ * @param types The publisher's own designator types, in the caller's order of precedence.
+ * The Dirección General del Catastro writes the type as the element text `1`, while Navarra
+ * and Gipuzkoa write the `LocatorDesignatorTypeValue/addressIdentifierGeneral` codelist URI.
+ */
+export function spanishHouseNumber(
+	byType: ReadonlyMap<string, readonly string[]>,
+	...types: readonly string[]
+): string | undefined {
+	const value = designator(byType, ...types)
+
+	if (!value || isSinNumero(value)) return undefined
+
+	return value
+}
+
+/**
+ * Whether a component reference's key states that the publisher holds no value for that component.
+ *
+ * A Spanish cadastre's feature identifier is a dotted path whose last segment is the value:
+ * the Cadastre's postal descriptors in Ceuta are `ES.SDGC.PD.55.101.51001` through
+ * `…51005`, and Gipuzkoa's are `ES.GFA.PD.056.20830`.
+ * Where the cadastre holds no postcode it writes the same identifier with that segment empty,
+ * `ES.SDGC.PD.55.101.` and `ES.GFA.PD.059.`, against no feature of that id.
+ *
+ * Those truncated references are the only unresolvable ones measured in either
+ * file: 70 of Ceuta's 8,126 addresses carry one, while its thoroughfare
+ * and administrative-unit references resolve 8,126 of 8,126 each.
+ * A reader taking every unresolved reference as an absence would also swallow a genuinely
+ * broken join, so the two are separated here and every other unresolved reference raises.
+ */
+export function referenceStatesNoValue(key: string): boolean {
+	return key.trim().endsWith(".")
+}
+
+/**
+ * What a cadastre's reader has read from one address, before it becomes a row.
+ *
+ * Each of the four readers resolves these from a different file layout, and `cadastralRow` turns any
+ * of them into the same row, so the layout stays in the adapter and the row shape has one home.
+ */
+export interface CadastralAddress {
+	/**
+	 * The thoroughfare name the address references, absent where the publisher writes none.
+	 */
+	street: string | undefined
+	/**
+	 * The house number this cadastre's own designator type states.
+	 */
+	house: string | undefined
+	/**
+	 * The postal descriptor's code.
+	 */
+	postcode: string | undefined
+	/**
+	 * The municipality-level administrative unit's name.
+	 */
+	locality: string | undefined
+	/**
+	 * The address-area name, which is a settlement within the municipality.
+	 */
+	settlement: string | undefined
+	/**
+	 * The publisher's own identifier for the address, which keys `source_id` where it exists.
+	 */
+	addressID: string | undefined
+}
+
+/**
+ * What distinguishes one cadastre's rows from another's.
+ *
+ * The license differs per publisher, so it is passed rather than derived: Gipuzkoa's register
+ * entry elects `CC-BY-SA-4.0` where the other three elect no share-alike obligation.
+ */
+export interface CadastralSource {
+	adapterID: string
+	license: string
+}
+
+/**
+ * One canonical row from what a Spanish cadastre states about one address.
+ *
+ * All four cadastres publish the INSPIRE Addresses theme, so they agree on how an
+ * address becomes a row even though each writes a different file layout.
+ * The municipality supplies the locality.
+ *
+ * The address area supplies a dependent locality only where the two differ.
+ * The line renders in Spain's order.
+ *
+ * Composing that in each adapter repeated 42 lines four times.
+ *
+ * Returns `undefined` in two cases: the address carries neither a municipality
+ * nor an address area, which would leave the row without a locality, or `formatAddressRow`
+ * returns `undefined` for the components it was given.
+ */
+export function cadastralRow(address: CadastralAddress, source: CadastralSource): CanonicalRow | undefined {
+	const place = address.locality ?? address.settlement
+
+	if (!place) return undefined
+
+	const components: CanonicalRow["components"] = {}
+
+	if (address.house) {
+		components.house_number = address.house
+	}
+
+	if (address.street) {
+		components.street = address.street
+	}
+
+	if (address.postcode) {
+		components.postcode = address.postcode
+	}
+
+	components.locality = place
+
+	// The address area is a settlement within the municipality where the two names differ,
+	// and repeats the municipality where they do not.
+	// Writing the repetition would state a dependent locality the publisher does not hold.
+	if (address.settlement && address.locality && address.settlement !== address.locality) {
+		components.dependent_locality = address.settlement
+	}
+
+	const rendered = formatAddressRow(components, "ES", { singleLine: true })
+
+	if (!rendered) return undefined
+
+	const { raw, components: aligned } = rendered
+
+	return {
+		raw,
+		components: aligned,
+		country: "ES",
+		locale: "es-ES",
+		// `register` is left to `runAdapter`, which stamps the adapter's own
+		// declaration onto a row that omits it.
+		// Setting it here would state per row what the adapter already states once.
+		source: source.adapterID,
+		source_id: address.addressID
+			? `${source.adapterID}-${address.addressID}`
+			: stableSourceID(source.adapterID, aligned),
+		corpus_version: "",
+		license: source.license,
+	}
+}