@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/cz/adapters/cuzk/adapter.ts

@@ -0,0 +1,320 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `cz-cuzk`: the INSPIRE Addresses theme ČÚZK publishes, read from its per-municipality GML.
+ *
+ * ČÚZK, the Czech Office for Surveying, Mapping and Cadastre, publishes one zipped GML per
+ * municipality through the ATOM service at `https://atom.cuzk.gov.cz/AD/AD.xml`. That service lists
+ * 6,258 dataset feeds. Each feed offers the same data in EPSG:5514 and EPSG:4258. The adapter reads
+ * a single archive or a directory of them, so a caller harvests municipality by municipality rather
+ * than waiting on a national file.
+ *
+ * Every address and every feature it references sit in the same archive, so this publisher is cheap
+ * to read. The `ad:component` hrefs look external, because ČÚZK writes an absolute WFS stored-query
+ * URL on `services.cuzk.cz`. The `Id=` parameter of each one matches a `gml:id` in the file at hand.
+ * Measured over `584282.xml`: 5,460 component references across 1,365 addresses, 0 of them reaching
+ * outside the archive. Only `ad:parcel` and `ad:building` point at genuinely separate themes, and
+ * the adapter reads neither.
+ *
+ * ČÚZK states one condition of use and states it uniformly. Every one of the 6,259 `rights` elements
+ * in the ATOM service document reads `žádné podmínky neplatí`, the INSPIRE controlled value
+ * `no conditions apply to access and use`. The address-source register elects that and records no
+ * license instrument, so no SPDX identifier exists to record and the rows store the controlled value
+ * itself. The register also holds a caution this adapter cannot resolve: that value states the
+ * absence of a restriction on access and use rather than granting copyright, so what it permits
+ * beyond access and use rests on Czech law.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import type { MarkupElement } from "@mailwoman/core/html/elements"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import {
+	adminUnitLevel,
+	componentHrefs,
+	componentJoinKey,
+	designator,
+	designatorsByType,
+	placeName,
+	postalDescriptorCode,
+	thoroughfareName,
+} from "#inspire/address"
+import { inspireGMLChunks } from "#inspire/archive"
+import { streamInspireRows } from "#inspire/stream"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const CZ_CUZK_ADAPTER_ID = "cz-cuzk"
+
+/**
+ * The jurisdiction this adapter emits.
+ */
+export const CZ_CUZK_COUNTRIES: readonly string[] = ["CZ"]
+
+/**
+ * The license recorded on every row.
+ *
+ * INSPIRE's controlled value, in the English the register renders it to, because ČÚZK
+ * names no license instrument and the register therefore writes no SPDX identifier.
+ */
+export const CZ_CUZK_LICENSE = "no conditions apply to access and use"
+
+/**
+ * The feature types an address references.
+ * The adapter indexes them as they arrive.
+ *
+ * `ad:Address` holds its number inline and is read by the driver rather than listed here.
+ * These four hold the street, the postcode and the two place names, and they are read in the same pass
+ * because an archive writes them interleaved with the addresses rather than ahead of them.
+ */
+const COMPONENT_TYPES = [
+	"ad:ThoroughfareName",
+	"ad:PostalDescriptor",
+	"ad:AdminUnitName",
+	"ad:AddressAreaName",
+] as const
+
+/**
+ * The member of a municipality's archive that holds the GML.
+ *
+ * ČÚZK names it for the municipality, as `584282.xml`, and ships one member per archive,
+ * so the extension selects it without the adapter knowing the code.
+ *
+ * ČÚZK's central directory records the member's size as the ZIP64 sentinel `0xFFFFFFFF`
+ * on at least one archive, so a reader trusting that slot reads the wrong length.
+ * `readZipEntry`, which {@linkcode inspireGMLChunks} uses, reads the ZIP64 extra field
+ * where the real length lives.
+ */
+const GML_MEMBER = /\.xml$/iu
+
+/**
+ * The series prefix a street name makes unnecessary.
+ *
+ * Czechia numbers a building in one of two series.
+ * A `číslo popisné` is the descriptive number an ordinary building holds, and a
+ * `číslo evidenční` numbers a recreational or temporary building in a separate series,
+ * so the same digits can name two buildings in one municipality.
+ */
+const DESCRIPTIVE_NUMBER_PREFIX = "č.p."
+
+/**
+ * The house number as ČÚZK writes it, including the series prefix wherever ČÚZK writes one.
+ *
+ * ČÚZK states the series as a `buildingIdentifierPrefix` designator on every address
+ * and renders it in three of four cases.
+ * Its own `ad:alternativeIdentifier` is the authority, and comparing this function's
+ * whole rendered line against that string over the 1,669 addresses of Židlochovice
+ * and Unkovice is what the table below was read from:
+ *
+ * | | street present | street absent |
+ * | --- | --- | --- |
+ * | `č.p.` | `Dezortova 803` | `č.p. 217, 66463 Unkovice` |
+ * | `č.ev.` | `Komenského č.ev. 502` | `č.ev. 38, 66463 Unkovice` |
+ *
+ * So the descriptive prefix is dropped only when a street name already places the number.
+ * A number with no street keeps its prefix, because the two series overlap
+ * and the digits by themselves would not say which building is meant.
+ *
+ * An earlier rule that dropped `č.p.` everywhere disagreed with the publisher on 254 of those 1,669 rows.
+ *
+ * A prefix other than `č.p.` is kept in every case, including one these two municipalities
+ * did not show, because the observed values come from 2 of 6,258 municipalities.
+ * An unrecognized prefix stays.
+ *
+ * That preserves a distinction the publisher drew rather than discarding it.
+ */
+function czechHouseNumber(
+	byType: ReadonlyMap<string, readonly string[]>,
+	options: { street: string | null }
+): string | null {
+	const number = designator(byType, "buildingIdentifier")
+
+	if (!number) return null
+
+	const prefix = designator(byType, "buildingIdentifierPrefix")
+
+	if (!prefix) return number
+
+	if (options.street && prefix === DESCRIPTIVE_NUMBER_PREFIX) return number
+
+	return `${prefix} ${number}`
+}
+
+export function createCzCuzkAdapter(): CorpusAdapter {
+	return {
+		id: CZ_CUZK_ADAPTER_ID,
+		defaultLicense: CZ_CUZK_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.CuzkInspireAddresses,
+		surface: SurfaceOrigin.Rendered,
+		description: "INSPIRE Addresses (ČÚZK): house-number-level addresses for Czechia, one zipped GML per municipality.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !CZ_CUZK_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(CZ_CUZK_ADAPTER_ID, CZ_CUZK_COUNTRIES, opts.country)
+			}
+
+			let emitted = 0
+
+			// One archive per municipality, each a document whose references resolve inside it,
+			// so the index is per archive rather than shared across the 6,258.
+			for (const archivePath of await archivePaths(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				for await (const row of streamInspireRows<MarkupElement>({
+					chunks: inspireGMLChunks(archivePath, GML_MEMBER),
+					componentElements: COMPONENT_TYPES,
+					// The subtree is kept, because one feature answers a street, a postcode
+					// or a place name depending on which type it is.
+					index: (feature) => feature,
+					compose: (address, referenced, final) => composeRow(address, referenced, { final }),
+					limit: opts.limit === undefined ? undefined : opts.limit - emitted,
+					signal: opts.signal,
+				})) {
+					yield row
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * One address's row, `null` when it holds too little to render, or `"deferred"`
+ * when a reference it cites is not indexed yet.
+ *
+ * The caller defers rather than drops because an unresolved reference and an absent one
+ * are different conditions, and only the archive's own ordering tells them apart.
+ * With `final` set the whole member has been read, so an unresolved reference is an
+ * absence the publisher is responsible for and the row is rendered without it.
+ */
+function composeRow(
+	address: MarkupElement,
+	referenced: ReadonlyMap<string, MarkupElement>,
+	options: { final?: boolean } = {}
+): CanonicalRow | null | "deferred" {
+	const byType = designatorsByType(address)
+
+	let street: string | null = null
+	let postcode: string | null = null
+	let locality: string | null = null
+	let settlement: string | null = null
+	let unresolved = false
+
+	for (const href of componentHrefs(address)) {
+		const key = componentJoinKey(href)
+
+		if (!key) continue
+
+		const feature = referenced.get(key)
+
+		if (!feature) {
+			// Only the four themes this adapter reads are expected in the archive.
+			// A parcel or building reference holds no `Id` the loop above accepted.
+			if (/^(?:TF|PD|AU|AA)\./u.test(key)) {
+				unresolved = true
+			}
+
+			continue
+		}
+
+		if (feature.name === "ad:ThoroughfareName") {
+			street = thoroughfareName(feature)
+		} else if (feature.name === "ad:PostalDescriptor") {
+			postcode = postalDescriptorCode(feature)
+		} else if (feature.name === "ad:AddressAreaName") {
+			settlement = placeName(feature)
+		} else if (feature.name === "ad:AdminUnitName" && adminUnitLevel(feature) === "4thOrder") {
+			locality = placeName(feature)
+		}
+	}
+
+	if (unresolved && !options.final) return "deferred"
+
+	// The street decides whether the number keeps its series prefix, so it is read first.
+	const house = czechHouseNumber(byType, { street })
+	const place = locality ?? settlement
+
+	if (!place) return null
+
+	if (!postcode && !street) return null
+
+	const components: CanonicalRow["components"] = {}
+
+	if (house) {
+		components.house_number = house
+	}
+
+	if (street) {
+		components.street = street
+	}
+
+	if (postcode) {
+		components.postcode = postcode
+	}
+
+	components.locality = place
+
+	// A `část obce`, the settlement part, is a dependent locality when the municipality is known.
+	if (settlement && locality && settlement !== locality) {
+		components.dependent_locality = settlement
+	}
+
+	const rendered = formatAddressRow(components, "CZ", { singleLine: true })
+
+	if (!rendered) return null
+
+	const { raw, components: aligned } = rendered
+	const seed = address.attributes["gml:id"]?.trim()
+
+	return {
+		raw,
+		components: aligned,
+		country: "CZ",
+		locale: "cs-CZ",
+		source: CZ_CUZK_ADAPTER_ID,
+		source_id: seed ? `${CZ_CUZK_ADAPTER_ID}-${seed}` : stableSourceID(CZ_CUZK_ADAPTER_ID, aligned),
+		corpus_version: "",
+		license: CZ_CUZK_LICENSE,
+	}
+}
+
+/**
+ * Every archive under `inputPath`, which is one `.zip` or a directory holding them.
+ *
+ * A directory is read because ČÚZK publishes 6,258 of these, one per municipality,
+ * and a caller harvesting the country points at the directory it fetched them into.
+ * A directory holding no archive raises rather than yielding zero rows, so a wrong
+ * path reports itself instead of reading as an empty publisher.
+ */
+async function archivePaths(inputPath: PathBuilderLike): Promise<readonly PathBuilderLike[]> {
+	const path = PathBuilder.from(inputPath)
+
+	if (path.basename().toLowerCase().endsWith(".zip")) return [path]
+
+	const names = await Globerator.from("*.zip", { cwd: path.toString(), absolute: false }).toSorted()
+
+	if (!names.length) {
+		throw new Error(`${CZ_CUZK_ADAPTER_ID} adapter: ${path.toString()} holds no .zip archive`)
+	}
+
+	return names.map((name) => path(name))
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const czCuzkAdapter = createCzCuzkAdapter()

package/lib/cz/recipes/pcfirst-preposition.ts

@@ -3,30 +3,29 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `cz-pcfirst-preposition` — the Czech pc-first prepositional-locality slice, fourth orthography
- *   of the #901 leading-name-boundary family. The #897 close-out read all 8 residual CZ rows as one
- *   class: a LEADING postcode mis-assigned as house_number while the multi-word "nad/pod/u X"
- *   locality shatters ("51244 Rokytnice nad Jizerou, Dolní Rokytnice 111" → street
- *   'RokytnicenadJizerou' + house_number '51244'). That leading-5-digit confusion is the
- *   anchor-pollution class whose decode-time OVERRIDE was correctly killed in #723 — this slice is
- *   the model-first fix as DATA: real prepositional localities in the order that breaks, so the
- *   model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
- *   cycle (the lesson); canonical and city-first keep the polarity balanced (the v1.9.9 lesson).
+ * `cz-pcfirst-preposition`: the Czech pc-first prepositional-locality recipe.
+ *
+ * A leading postcode is mis-assigned as a house number while a multi-word "nad/pod/u X" locality
+ * shatters. This recipe supplies real prepositional localities in the order that breaks, so the
+ * model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
+ * cycle and the canonical and city-first orders keep the polarity balanced.
  */
 
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import { alignAndWrite, readTuples, requireRegister, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 /**
- * The order-cycle slot for the STREET-LESS form (`«city» «pc», Česko`) — the exact surface of the
- * `cz-full-praha-100-00` board row, whose absence from the street-bearing orders was the v4.5.0 no-promote's measured
- * gap.
+ * The order-cycle slot for the street-less form (`«city» «pc», Česko`).
+ *
+ * This is the surface of the `cz-full-praha-100-00` board row.
+ * The street-containing orders do not cover it.
  */
 const STREETLESS_ORDER = 3
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * Recipe registered with the corpus builder.
  */
 export const czPcFirstPrepositionRecipe: CorpusRecipe = {
 	name: "cz-pcfirst-preposition",
@@ -39,6 +38,11 @@ export const czPcFirstPrepositionRecipe: CorpusRecipe = {
 		let emitted = 0
 		let skipped = 0
 
+		const CZ_PCFIRST_PROVENANCE = {
+			register: requireRegister(opts, "cz-pcfirst-preposition"),
+			surface: SurfaceOrigin.Composed,
+		}
+
 		for await (const t of readTuples(opts.input!)) {
 			read++
 			const street = String(t.street ?? "").trim()
@@ -53,10 +57,10 @@ export const czPcFirstPrepositionRecipe: CorpusRecipe = {
 			}
 
 			const order = read % 4
-			// The OFFICIAL Czech rendering spaces the PSČ as `NNN NN` ('512 44'); OpenAddresses stores it
-			// unspaced ('51244'), and a model trained only on the source form reads the spaced surface as
-			// house_number + garbage (the 'Praha 100 00' mangle). Alternate the two renderings so both
-			// orthographies are attested — the LABEL is the postcode either way.
+			// The official Czech rendering spaces the PSČ as `NNN NN` ('512 44')
+			// while OpenAddresses stores it unspaced ('51244').
+			// Alternate the two renderings so both orthographies are attested.
+			// Label the postcode in either form.
 			const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
 			const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
 			let raw: string
@@ -73,15 +77,16 @@ export const czPcFirstPrepositionRecipe: CorpusRecipe = {
 			} else if (order === 2) {
 				raw = `${city}, ${postcodeSurface}, ${street} ${number}`
 			} else {
-				// order === STREETLESS_ORDER: the street-less form — `«city» «pc», Česko` — the exact surface of the
-				// cz-full-praha-100-00 board row. The v4.5.0 no-promote receipt measured the gap: every
-				// prior order was street-bearing, so the model never saw a spaced PSČ beside a bare
-				// locality and mangled 'Praha 100 00, Czechia' into house_number spans. Street/number
-				// stay OUT of the components for this form (they are not in the surface).
+				// The street-less form `«city» «pc», Česko` omits street and number from the
+				// components because the surface does not include them.
 				raw = `${city} ${postcodeSurface}, Česko`
 			}
 
-			const source_id = sliceSourceID("synth-cz-pcfirst-preposition", {
+			// One resolution for both, because `source_id` includes the source as its prefix
+			// and a pair that disagreed would name a source no row of this output declares.
+			const source = defaultRecipeSource("synth-cz-pcfirst-preposition")
+
+			const source_id = recipeSourceID(source, {
 				...components,
 				o: String(order),
 				s: spaced ? "1" : "0",
@@ -93,14 +98,22 @@ export const czPcFirstPrepositionRecipe: CorpusRecipe = {
 				components,
 				country: "CZ",
 				locale: "cs-CZ",
-				source: "synth-cz-pcfirst-preposition",
+				source,
 				source_id,
 				corpus_version: "0.10.0",
 				license:
 					"Synthetic — cz-pcfirst-preposition; (street, number, postcode, city) from OpenAddresses CZ (per-source attribution in the model card)",
 			}
 
-			if (alignAndWrite(write, canonical, "cz-pcfirst-preposition")) {
+			// Per row, because the id names this tuple's record.
+			// A tuples file written before `sourceID` existed has none.
+			// `null` means the recipe had no id to forward.
+			if (
+				alignAndWrite(write, canonical, "cz-pcfirst-preposition", {
+					...CZ_PCFIRST_PROVENANCE,
+					baseSourceID: t.sourceID ?? null,
+				})
+			) {
 				emitted++
 			} else {
 				skipped++