@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/cn/recipes/organizational-units.ts

@@ -0,0 +1,178 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Labels Chinese rows whose settlement ends in an organizational unit chain.
+ *
+ *   The unit grammar tags the whole chain as `locality_unit` and its head unit as `dependent_locality`. Input is JSONL
+ *   with `{ raw, country }`. Rows are aligned per character for the CJK model.
+ */
+
+import { stringifyJSON } from "@mailwoman/core/json"
+import { splitCNUnitChain } from "@mailwoman/core/locale/zh-cn-units"
+import { TextSpliterator } from "spliterator"
+
+import { type CorpusRecipe, readTuples, recipeSourceID } from "#recipes/scaffold"
+import { alignRow } from "#utils/align"
+import { cjkAwareTokenizer } from "#utils/tokenize"
+
+/**
+ * Matches the leading run of Han characters and digits.
+ */
+const LEADING_HAN = /^[\p{Script=Han}〇\d]+/u
+
+/**
+ * Administrative prefixes, matched in order from broadest to narrowest.
+ */
+const ADMIN_PREFIXES: ReadonlyArray<readonly [pattern: RegExp, tag: "region" | "locality" | "subregion"]> = [
+	[/^(.+?(?:省|自治区))/u, "region"],
+	[/^(.+?市)/u, "locality"],
+	[/^(.+?(?:自治县|县|旗))/u, "subregion"],
+]
+
+/**
+ * Labels the components of a row, or returns `null` when the row has no unit chain.
+ */
+export function labelCNOrganizationalRow(raw: string): Record<string, string> | null {
+	const han = LEADING_HAN.exec(raw)?.[0]
+
+	if (!han) return null
+
+	const components: Record<string, string> = {}
+	let rest = han
+
+	for (const [pattern, tag] of ADMIN_PREFIXES) {
+		const match = pattern.exec(rest)
+
+		// A prefix is taken only when text remains after it for the settlement.
+		if (match && match[1]!.length < rest.length) {
+			components[tag] = match[1]!
+			rest = rest.slice(match[1]!.length)
+		}
+	}
+
+	const split = splitCNUnitChain(rest)
+
+	if (!split) return null
+
+	if (split.head) {
+		components.dependent_locality = split.head
+	}
+
+	components.locality_unit = split.chain
+
+	const tail = raw.slice(han.length).trim()
+
+	if (tail) {
+		// A Latin tail becomes a region and a trailing "China" country.
+		const segments = tail.includes(",")
+			? TextSpliterator.from(tail, { delimiter: "," }).toArray()
+			: tailWithoutCommas(tail)
+
+		for (const segment of segments) {
+			if (/^china$/iu.test(segment)) {
+				components.country = segment
+			} else if (!components.region && !/^\d+$/u.test(segment)) {
+				components.region = segment
+			}
+		}
+	}
+
+	return components
+}
+
+/**
+ * Splits a trailing `China` off a tail without commas.
+ * Any other tail stays one segment.
+ */
+function tailWithoutCommas(tail: string): string[] {
+	const words = tail.split(/\s+/u).filter((word) => word.length)
+	const last = words.at(-1)
+
+	if (last && /^china$/iu.test(last) && words.length > 1) {
+		return [words.slice(0, -1).join(" "), last]
+	}
+
+	return [words.join(" ")]
+}
+
+const SOURCE = "coarse-placer-cn-units"
+
+/**
+ * The recipe registered with the corpus builder.
+ */
+export const cnOrganizationalUnitsRecipe: CorpusRecipe = {
+	name: "cn-organizational-units",
+	description:
+		"CN rows whose settlement is an organizational ladder (分场/队/连/组), labeled by the suffix grammar as one locality_unit span, per-character BIO",
+	mode: "tuples",
+	options: [],
+	async run(opts, write) {
+		if (!opts.input) {
+			throw new Error("cn-organizational-units: --input <rows.jsonl> is required ({ raw, country } rows)")
+		}
+
+		const tokenizer = cjkAwareTokenizer()
+		let emitted = 0
+		let skipped = 0
+		let quarantined = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			const raw = typeof tuple["raw"] === "string" ? tuple["raw"].trim() : ""
+			const country = typeof tuple["country"] === "string" ? tuple["country"].toUpperCase() : "CN"
+
+			if (!raw || country !== "CN") {
+				skipped++
+
+				continue
+			}
+
+			const components = labelCNOrganizationalRow(raw)
+
+			if (!components) {
+				skipped++
+
+				continue
+			}
+
+			if (opts.golden) {
+				write(stringifyJSON({ raw, components, country: "CN", locale: "zh-CN" }))
+
+				emitted++
+
+				continue
+			}
+
+			const canonical = {
+				raw,
+				components,
+				country: "CN",
+				locale: "zh-CN",
+				source: opts.sourceName ?? SOURCE,
+				source_id: recipeSourceID(opts.sourceName ?? SOURCE, { raw }),
+				corpus_version: "0.4.0",
+				license:
+					"CC-BY-4.0 — GeoNames populated places, INFERRED from the `<name>, <admin1>, <country>` row shape; data/coarse-placer carries no per-row source",
+			}
+
+			// Every component must appear verbatim in `raw`.
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0], { tokenizer, maxEditDistance: 0 })
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				quarantined++
+
+				continue
+			}
+
+			write(stringifyJSON({ ...aligned.row, synth_method: "cn-organizational-units", synth_base_id: null }))
+
+			emitted++
+		}
+
+		console.error(
+			`  cn-organizational-units: ${emitted} labeled, ${skipped} without a chain, ${quarantined} quarantined`
+		)
+
+		return { emitted, skipped, quarantined }
+	},
+}

package/lib/cz/adapters/cuzk/adapter.ts

@@ -0,0 +1,317 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `cz-cuzk`: the INSPIRE Addresses theme ČÚZK publishes, read from its per-municipality GML.
+ *
+ * ČÚZK, the Czech Office for Surveying, Mapping and Cadastre, publishes one zipped GML per
+ * municipality through the ATOM service at `https://atom.cuzk.gov.cz/AD/AD.xml`, which lists 6,258
+ * dataset feeds. Each feed offers the same data in EPSG:5514 and EPSG:4258. The adapter reads a
+ * single archive or a directory of them, so a caller harvests municipality by municipality rather
+ * than waiting on a national file.
+ *
+ * Every address and every feature it references sit in the same archive, which is what makes this
+ * publisher cheap to read. The `ad:component` hrefs look external, because ČÚZK writes an absolute
+ * WFS stored-query URL on `services.cuzk.cz`, and the `Id=` parameter of each one matches a
+ * `gml:id` in the file at hand. Measured over `584282.xml`: 5,460 component references across 1,365
+ * addresses, 0 of them reaching outside the archive. Only `ad:parcel` and `ad:building` point at
+ * genuinely separate themes, and the adapter reads neither.
+ *
+ * ČÚZK states one condition of use and states it uniformly. Every one of the 6,259 `rights` elements
+ * in the ATOM service document reads `žádné podmínky neplatí`, which renders INSPIRE's controlled
+ * value `no conditions apply to access and use`. The address-source register elects that and names
+ * no license instrument, so no SPDX identifier exists to record and the rows carry the controlled
+ * value itself. The register also carries a caution this adapter cannot resolve: that value states
+ * the absence of a restriction on access and use rather than granting copyright, so what it permits
+ * beyond access and use rests on Czech law.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import type { MarkupElement } from "@mailwoman/core/html/elements"
+import { PathBuilder, type PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceID } from "#adapters/source-id"
+import {
+	adminUnitLevel,
+	componentHrefs,
+	componentJoinKey,
+	designator,
+	designatorsByType,
+	placeName,
+	postalDescriptorCode,
+	thoroughfareName,
+} from "#inspire/address"
+import { inspireGMLChunks } from "#inspire/archive"
+import { streamInspireRows } from "#inspire/stream"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+
+/**
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
+ */
+export const CZ_CUZK_ADAPTER_ID = "cz-cuzk"
+
+/**
+ * The jurisdiction this adapter emits.
+ */
+export const CZ_CUZK_COUNTRIES: readonly string[] = ["CZ"]
+
+/**
+ * The license recorded on every row.
+ *
+ * INSPIRE's controlled value, in the English the register renders it to, because ČÚZK
+ * names no license instrument and the register therefore writes no SPDX identifier.
+ */
+export const CZ_CUZK_LICENSE = "no conditions apply to access and use"
+
+/**
+ * The feature types an address references, which the adapter indexes as they arrive.
+ *
+ * `ad:Address` carries its number inline and is read by the driver rather than listed here.
+ * These four hold the street, the postcode and the two place names, and they are read in the same pass
+ * because an archive writes them interleaved with the addresses rather than ahead of them.
+ */
+const COMPONENT_TYPES = [
+	"ad:ThoroughfareName",
+	"ad:PostalDescriptor",
+	"ad:AdminUnitName",
+	"ad:AddressAreaName",
+] as const
+
+/**
+ * The member of a municipality's archive that holds the GML.
+ *
+ * ČÚZK names it for the municipality, as `584282.xml`, and ships one member per archive,
+ * so the extension selects it without the adapter knowing the code.
+ *
+ * ČÚZK's central directory records the member's size as the ZIP64 sentinel `0xFFFFFFFF`
+ * on at least one archive, so a reader trusting that slot reads the wrong length.
+ * `readZipEntry`, which {@linkcode inspireGMLChunks} uses, reads the ZIP64 extra field
+ * where the real length lives.
+ */
+const GML_MEMBER = /\.xml$/iu
+
+/**
+ * The series prefix a street name makes unnecessary.
+ *
+ * Czechia numbers a building in one of two series.
+ * A `číslo popisné` is the descriptive number an ordinary building carries, and a
+ * `číslo evidenční` numbers a recreational or temporary building in a separate series,
+ * so the same digits can name two buildings in one municipality.
+ */
+const DESCRIPTIVE_NUMBER_PREFIX = "č.p."
+
+/**
+ * The house number as ČÚZK writes it, including the series prefix wherever ČÚZK writes one.
+ *
+ * ČÚZK states the series as a `buildingIdentifierPrefix` designator on every address
+ * and renders it in three of four cases.
+ * Its own `ad:alternativeIdentifier` is the authority, and comparing this function's
+ * whole rendered line against that string over the 1,669 addresses of Židlochovice
+ * and Unkovice is what the table below was read from:
+ *
+ * | | street present | street absent |
+ * | --- | --- | --- |
+ * | `č.p.` | `Dezortova 803` | `č.p. 217, 66463 Unkovice` |
+ * | `č.ev.` | `Komenského č.ev. 502` | `č.ev. 38, 66463 Unkovice` |
+ *
+ * So the descriptive prefix is dropped only when a street name already places the number.
+ * A number standing alone keeps its prefix, because the two series overlap
+ * and the digits alone would not say which building is meant.
+ *
+ * An earlier rule that dropped `č.p.` everywhere disagreed with the publisher on 254 of those 1,669 rows.
+ *
+ * A prefix other than `č.p.` is kept in every case, including one these two municipalities did
+ * not carry, because the observed values come from 2 of 6,258 municipalities and keeping an
+ * unrecognized prefix preserves a distinction the publisher drew rather than discarding it.
+ */
+function czechHouseNumber(
+	byType: ReadonlyMap<string, readonly string[]>,
+	options: { street?: string }
+): string | undefined {
+	const number = designator(byType, "buildingIdentifier")
+
+	if (!number) return undefined
+
+	const prefix = designator(byType, "buildingIdentifierPrefix")
+
+	if (!prefix) return number
+
+	if (options.street && prefix === DESCRIPTIVE_NUMBER_PREFIX) return number
+
+	return `${prefix} ${number}`
+}
+
+export function createCzCuzkAdapter(): CorpusAdapter {
+	return {
+		id: CZ_CUZK_ADAPTER_ID,
+		defaultLicense: CZ_CUZK_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.CuzkInspireAddresses,
+		surface: SurfaceOrigin.Rendered,
+		description: "INSPIRE Addresses (ČÚZK): house-number-level addresses for Czechia, one zipped GML per municipality.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !CZ_CUZK_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(CZ_CUZK_ADAPTER_ID, CZ_CUZK_COUNTRIES, opts.country)
+			}
+
+			let emitted = 0
+
+			// One archive per municipality, each a document whose references resolve inside it,
+			// so the index is per archive rather than shared across the 6,258.
+			for (const archivePath of await archivePaths(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				for await (const row of streamInspireRows<MarkupElement>({
+					chunks: inspireGMLChunks(archivePath, GML_MEMBER),
+					componentElements: COMPONENT_TYPES,
+					// The subtree is kept, because one feature answers a street, a postcode
+					// or a place name depending on which type it is.
+					index: (feature) => feature,
+					compose: (address, referenced, final) => composeRow(address, referenced, { final }),
+					limit: opts.limit === undefined ? undefined : opts.limit - emitted,
+					signal: opts.signal,
+				})) {
+					yield row
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * One address's row, `undefined` when it carries too little to render, or `"deferred"`
+ * when a reference it names is not indexed yet.
+ *
+ * Deferring rather than dropping matters because an unresolved reference and an absent
+ * one are different conditions, and only the archive's own ordering tells them apart.
+ * With `final` set the whole member has been read, so an unresolved reference is an
+ * absence the publisher is responsible for and the row is rendered without it.
+ */
+function composeRow(
+	address: MarkupElement,
+	referenced: ReadonlyMap<string, MarkupElement>,
+	options: { final?: boolean } = {}
+): CanonicalRow | undefined | "deferred" {
+	const byType = designatorsByType(address)
+
+	let street: string | undefined
+	let postcode: string | undefined
+	let locality: string | undefined
+	let settlement: string | undefined
+	let unresolved = false
+
+	for (const href of componentHrefs(address)) {
+		const key = componentJoinKey(href)
+
+		if (!key) continue
+
+		const feature = referenced.get(key)
+
+		if (!feature) {
+			// Only the four themes this adapter reads are expected in the archive.
+			// A parcel or building reference carries no `Id` the loop above accepted.
+			if (/^(?:TF|PD|AU|AA)\./u.test(key)) {
+				unresolved = true
+			}
+
+			continue
+		}
+
+		if (feature.name === "ad:ThoroughfareName") {
+			street = thoroughfareName(feature)
+		} else if (feature.name === "ad:PostalDescriptor") {
+			postcode = postalDescriptorCode(feature)
+		} else if (feature.name === "ad:AddressAreaName") {
+			settlement = placeName(feature)
+		} else if (feature.name === "ad:AdminUnitName" && adminUnitLevel(feature) === "4thOrder") {
+			locality = placeName(feature)
+		}
+	}
+
+	if (unresolved && !options.final) return "deferred"
+
+	// The street decides whether the number keeps its series prefix, so it is read first.
+	const house = czechHouseNumber(byType, { street })
+	const place = locality ?? settlement
+
+	if (!place) return undefined
+
+	if (!postcode && !street) return undefined
+
+	const components: CanonicalRow["components"] = {}
+
+	if (house) {
+		components.house_number = house
+	}
+
+	if (street) {
+		components.street = street
+	}
+
+	if (postcode) {
+		components.postcode = postcode
+	}
+
+	components.locality = place
+
+	// A `část obce`, the settlement part, is a dependent locality when the municipality is known.
+	if (settlement && locality && settlement !== locality) {
+		components.dependent_locality = settlement
+	}
+
+	const rendered = formatAddressRow(components, "CZ", { singleLine: true })
+
+	if (!rendered) return undefined
+
+	const { raw, components: aligned } = rendered
+	const seed = address.attributes["gml:id"]?.trim()
+
+	return {
+		raw,
+		components: aligned,
+		country: "CZ",
+		locale: "cs-CZ",
+		source: CZ_CUZK_ADAPTER_ID,
+		source_id: seed ? `${CZ_CUZK_ADAPTER_ID}-${seed}` : stableSourceID(CZ_CUZK_ADAPTER_ID, aligned),
+		corpus_version: "",
+		license: CZ_CUZK_LICENSE,
+	}
+}
+
+/**
+ * Every archive under `inputPath`, which is one `.zip` or a directory holding them.
+ *
+ * A directory is read because ČÚZK publishes 6,258 of these, one per municipality,
+ * and a caller harvesting the country points at the directory it fetched them into.
+ * A directory holding no archive raises rather than yielding zero rows, so a wrong
+ * path reports itself instead of reading as an empty publisher.
+ */
+async function archivePaths(inputPath: PathBuilderLike): Promise<readonly PathBuilderLike[]> {
+	const path = PathBuilder.from(inputPath)
+
+	if (path.basename().toLowerCase().endsWith(".zip")) return [path]
+
+	const names = await Globerator.from("*.zip", { cwd: path.toString(), absolute: false }).toSorted()
+
+	if (!names.length) {
+		throw new Error(`${CZ_CUZK_ADAPTER_ID} adapter: ${path.toString()} holds no .zip archive`)
+	}
+
+	return names.map((name) => path(name))
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const czCuzkAdapter = createCzCuzkAdapter()

package/lib/cz/recipes/pcfirst-preposition.ts

@@ -0,0 +1,125 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `cz-pcfirst-preposition`: the Czech pc-first prepositional-locality recipe.
+ *
+ * A leading postcode is mis-assigned as a house number while a multi-word "nad/pod/u X" locality
+ * shatters. This recipe supplies real prepositional localities in the order that breaks, so the
+ * model learns that a leading postcode before a multi-word name is a postcode. pc-first leads the
+ * cycle and the canonical and city-first orders keep the polarity balanced.
+ */
+
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+
+import { alignAndWrite, readTuples, requireRegister, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
+/**
+ * The order-cycle slot for the street-less form (`«city» «pc», Česko`).
+ *
+ * This is the surface of the `cz-full-praha-100-00` board row.
+ * The street-containing orders do not cover it.
+ */
+const STREETLESS_ORDER = 3
+
+/**
+ * Recipe registered with the corpus builder.
+ */
+export const czPcFirstPrepositionRecipe: CorpusRecipe = {
+	name: "cz-pcfirst-preposition",
+	description:
+		"CZ pc-first + prepositional locality (#901 family): '«pc» «city nad X», «st» «n»' — the #723 class as data",
+	mode: "tuples",
+	async run(opts, write) {
+		makeMulberry32(opts.seed)
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		const CZ_PCFIRST_PROVENANCE = {
+			register: requireRegister(opts, "cz-pcfirst-preposition"),
+			surface: SurfaceOrigin.Composed,
+		}
+
+		for await (const t of readTuples(opts.input!)) {
+			read++
+			const street = String(t.street ?? "").trim()
+			const city = String(t.locality ?? "").trim()
+			const number = String(t.number ?? "").trim()
+			const postcode = String(t.postcode ?? "").trim()
+
+			if (!street || !city || !number || !postcode) {
+				skipped++
+
+				continue
+			}
+
+			const order = read % 4
+			// The official Czech rendering spaces the PSČ as `NNN NN` ('512 44')
+			// while OpenAddresses stores it unspaced ('51244').
+			// Alternate the two renderings so both orthographies are attested.
+			// Label the postcode in either form.
+			const spaced = read % 2 === 0 && /^\d{5}$/.test(postcode)
+			const postcodeSurface = spaced ? `${postcode.slice(0, 3)} ${postcode.slice(3)}` : postcode
+			let raw: string
+
+			const components: Record<string, string> =
+				order === STREETLESS_ORDER
+					? { locality: city, postcode: postcodeSurface, country: "Česko" }
+					: { street, house_number: number, postcode: postcodeSurface, locality: city }
+
+			if (order === 0) {
+				raw = `${postcodeSurface} ${city}, ${street} ${number}`
+			} else if (order === 1) {
+				raw = `${street} ${number}, ${postcodeSurface} ${city}`
+			} else if (order === 2) {
+				raw = `${city}, ${postcodeSurface}, ${street} ${number}`
+			} else {
+				// The street-less form `«city» «pc», Česko` omits street and number from the
+				// components because the surface does not include them.
+				raw = `${city} ${postcodeSurface}, Česko`
+			}
+
+			// One resolution for both, because `source_id` includes the source as its prefix
+			// and a pair that disagreed would name a source no row of this output declares.
+			const source = defaultRecipeSource("synth-cz-pcfirst-preposition")
+
+			const source_id = recipeSourceID(source, {
+				...components,
+				o: String(order),
+				s: spaced ? "1" : "0",
+				v: String(read),
+			})
+
+			const canonical = {
+				raw,
+				components,
+				country: "CZ",
+				locale: "cs-CZ",
+				source,
+				source_id,
+				corpus_version: "0.10.0",
+				license:
+					"Synthetic — cz-pcfirst-preposition; (street, number, postcode, city) from OpenAddresses CZ (per-source attribution in the model card)",
+			}
+
+			// Per row, because the id names this tuple's record.
+			// A tuples file written before `sourceID` existed has none.
+			// `null` means the recipe had no id to forward.
+			if (
+				alignAndWrite(write, canonical, "cz-pcfirst-preposition", {
+					...CZ_PCFIRST_PROVENANCE,
+					baseSourceID: t.sourceID ?? null,
+				})
+			) {
+				emitted++
+			} else {
+				skipped++
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}