@mailwoman/corpus 10.6.0 → 10.7.0

package/data/us-situs-sources.json

@@ -0,0 +1,128 @@
+{
+	"$comment": "Maps each `source` value of the US situs databases (address-points-us-*.db under the data root) to the register rows its data comes from. The house-venue tuple reader in corpus-python/src/mailwoman_train/countries/us/tuples.py keeps a row only when every register row its value maps to holds an elected decision permitting transform and train. `overture:OpenAddresses/<state>/<body>` values map to that body's own row, because OpenAddresses does not relicense what it collects. A value with no entry, such as the bare `openaddresses` of the Hawaii database, which names no body, is not admitted.",
+	"sources": {
+		"overture:NAD": ["us-property-building-1", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Alameda County": ["us-property-building-27", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Alpine County": ["us-property-building-126", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Amador County": ["us-property-building-112", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Butte County": ["us-property-building-71", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Calaveras County": ["us-property-building-101", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Colusa County": ["us-property-building-122", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Contra Costa County": ["us-property-building-35", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/County of Los Angeles": ["us-property-building-10", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Del Norte County": ["us-property-building-118", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/El Dorado County": ["us-property-building-78", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Fresno County": ["us-property-building-37", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Glenn County": ["us-property-building-121", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Humboldt County": ["us-property-building-97", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Imperial County": ["us-property-building-90", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Inyo County": ["us-property-building-119", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Kern County": ["us-property-building-46", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Kings County": ["us-property-building-98", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Lake County": ["us-property-building-93", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Lassen County": ["us-property-building-124", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Madera County": ["us-property-building-95", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Marin County": ["us-property-building-3", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Mariposa County": ["us-property-building-120", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Mendocino County": ["us-property-building-99", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Merced County": ["us-property-building-76", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Modoc County": ["us-property-building-123", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Mono County": ["us-property-building-113", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Monterey County": ["us-property-building-67", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Napa County": ["us-property-building-91", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Nevada County": ["us-property-building-92", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Orange County": ["us-property-building-15", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Placer County": ["us-property-building-54", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Plumas County": ["us-property-building-109", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Riverside County": ["us-property-building-20", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Sacramento County": ["us-property-building-25", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Benito County": ["us-property-building-110", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Bernardino County": ["us-property-building-19", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Diego County": ["us-property-building-17", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Francisco County": ["us-property-building-36", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Joaquin County": ["us-property-building-48", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Luis Obispo County": ["us-property-building-66", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/San Mateo County": ["us-property-building-49", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Santa Barbara County": ["us-property-building-61", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Santa Clara County": ["us-property-building-30", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Santa Cruz County": ["us-property-building-75", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Shasta County": ["us-property-building-86", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Sierra County": ["us-property-building-125", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Siskiyou County": ["us-property-building-111", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Solano County": ["us-property-building-58", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Sonoma County": ["us-property-building-50", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Stanislaus County": ["us-property-building-59", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Sutter County": ["us-property-building-100", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Tehama County": ["us-property-building-106", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Trinity County": ["us-property-building-117", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Tulare County": ["us-property-building-60", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Tuolumne County": ["us-property-building-105", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Yolo County": ["us-property-building-77", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CA/Yuba County": ["us-property-building-103", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/CO/Governor's Office of Information Technology": [
+			"us-property-building-11",
+			"us-geodata-aggregate-1"
+		],
+		"overture:OpenAddresses/FL/Alachua County": ["us-property-building-62", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Baker County": ["us-property-building-116", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Bay County": ["us-property-building-65", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Brevard County": ["us-property-building-40", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Broward County": ["us-property-building-31", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Charlotte County": ["us-property-building-41", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Citrus County": ["us-property-building-56", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Clay County": ["us-property-building-83", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Collier County": ["us-property-building-43", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Columbia County": ["us-property-building-102", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Desoto County": ["us-property-building-114", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Duval County": ["us-property-building-26", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Flagler County": ["us-property-building-82", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Florida Department of Revenue": ["us-property-building-32", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Hardee County": ["us-property-building-115", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Hendry County": ["us-property-building-104", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Hernando County": ["us-property-building-69", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Highlands County": ["us-property-building-64", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Hillsborough County": ["us-property-building-22", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Indian River County": ["us-property-building-74", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Lake County": ["us-property-building-47", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Lee County": ["us-property-building-24", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Leon County": ["us-property-building-63", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Manatee County": ["us-property-building-42", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Marion County": ["us-property-building-68", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Martin County": ["us-property-building-79", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Miami-Dade County": ["us-property-building-14", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Nassau County": ["us-property-building-96", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Okaloosa County": ["us-property-building-70", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Orange County": ["us-property-building-23", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Osceola County": ["us-property-building-55", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Palm Beach County": ["us-property-building-21", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Pasco County": ["us-property-building-38", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Pinellas County": ["us-property-building-28", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Polk County": ["us-property-building-33", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Putnam County": ["us-property-building-84", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Santa Rosa County": ["us-property-building-80", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Sarasota County": ["us-property-building-39", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Seminole County": ["us-property-building-51", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/St. Johns County": ["us-property-building-57", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/St. Lucie County": ["us-property-building-53", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Sumter County": ["us-property-building-73", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Volusia County": ["us-property-building-44", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/FL/Walton County": ["us-property-building-89", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/GA/Athens-Clarke County GIS": ["us-property-building-87", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/GA/DeKalb County": ["us-property-building-34", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/ID/City of Boise": ["us-property-building-45", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MA/MassGIS": ["us-property-building-9", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MI/Allegan County": ["us-property-building-94", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MI/Kent County": ["us-property-building-52", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MI/Muskegon County": ["us-property-building-88", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MI/Oakland County": ["us-property-building-29", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MN/Department of Public Safety": ["us-property-building-12", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MS/Desoto County": ["us-property-building-85", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/MS/MARIS": ["us-property-building-16", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/NV/Carson City": ["us-property-building-108", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/NY/NYC Open Data": ["us-property-building-18", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/OR/Jackson County": ["us-property-building-72", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/OR/State of Oregon": ["us-property-building-13", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/PA/City of Scranton GIS": ["us-property-building-81", "us-geodata-aggregate-1"],
+		"overture:OpenAddresses/WI/Waupaca County": ["us-property-building-107", "us-geodata-aggregate-1"]
+	}
+}

package/lib/adapters/cordis/adapter.ts

@@ -67,7 +67,7 @@
  *
  * A country whose codex `layoutForCountry` is `null` has no rendering to write the row in.
  * The row is refused with {@linkcode CORDISRefusal.CountryNoLayout} rather than rendered in another
- * country's order: 353 records over 37 countries, Ghana's 77 the most.
+ * country's order.
  *
  * ## The street line
  *

package/lib/adapters/libpostal-dictionary.ts

@@ -22,16 +22,36 @@ import { TextSpliterator } from "spliterator"
  * The largest dictionary is 8.4 KB, and each caller loads it once per process at module load.
  */
 export async function loadLibpostalDictionary(language: string, filename: string): Promise<Set<string>> {
+	return new Set((await loadLibpostalCanonicalForms(language, filename)).keys())
+}
+
+/**
+ * Loads one curated libpostal dictionary as a map from each lower-cased form
+ * to its line's canonical spelling.
+ *
+ * `ул.` and `улица` both map to `улица`, so a count of distinct canonical values
+ * counts street types rather than their spellings.
+ * A form two lines share keeps the first line's canonical spelling.
+ */
+export async function loadLibpostalCanonicalForms(language: string, filename: string): Promise<Map<string, string>> {
 	const text = await readLocalTextFile(resourceDictionaryPath("libpostal", language, filename))
-	const set = new Set<string>()
+	const forms = new Map<string, string>()
 
 	for (const line of TextSpliterator.from(text)) {
 		if (line.startsWith("#")) continue
 
+		let canonical: string | null = null
+
 		for (const form of TextSpliterator.from(line, { delimiter: "|" })) {
-			set.add(form.toLowerCase())
+			const lower = form.toLowerCase()
+
+			canonical ??= lower
+
+			if (!forms.has(lower)) {
+				forms.set(lower, canonical)
+			}
 		}
 	}
 
-	return set
+	return forms
 }

package/lib/adapters/osm/adapter.ts

@@ -11,6 +11,7 @@
  */
 
 import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { countryPostcodeFormat, matchCountryPostcode } from "@mailwoman/codex/postcode/country-formats"
 import { tryParsingJSON } from "@mailwoman/core/json"
 import { extractDelimited } from "@mailwoman/core/scripting/arguments"
 import { stripCombiningMarks } from "@mailwoman/normalize/fold"
@@ -71,6 +72,15 @@ interface OSMCorpusRow {
 	district?: string | null
 	province?: string | null
 	country?: string | null
+	/**
+	 * The containing administrative area's name, which `emit-corpus-jsonl --locality-level`
+	 * writes for a record without `addr:city`.
+	 */
+	boundary_locality?: string | null
+	/**
+	 * The same for a record without `addr:suburb`, from `--dependent-locality-level`.
+	 */
+	boundary_dependent_locality?: string | null
 }
 
 const MAX_STREET_WORDS = 8
@@ -150,10 +160,13 @@ function parseLine(line: string): OSMCorpusRow | null {
  * A row whose `addr:street` holds a line rather than a name stays refused:
  * the mapper supplied a street, and the value is a line.
  *
+ * A postcode is kept when the whole value has `country`'s postcode format.
+ *
  * Each refusal and each discarded component increments its reason in `dropped`.
  */
 export function componentsForOSMRow(
 	row: OSMCorpusRow,
+	country: string,
 	dropped?: Pick<AdapterOptions, "dropped">
 ): CanonicalRow["components"] | null {
 	const tally = dropped ?? {}
@@ -189,10 +202,19 @@ export function componentsForOSMRow(
 
 	const postcode = clean(row.postcode)
 
-	if (postcode && /^\d{4,6}$/u.test(postcode)) {
-		components.postcode = postcode
-	} else if (postcode) {
-		countDropped(tally, "component:postcode:not-4-to-6-digits")
+	if (postcode) {
+		// The country's own format decides, so Argentina's `C1425ABC`, Brunei's `BS8811` and Chile's
+		// seven digits stay postcodes, and a value outside the country's form is dropped by name.
+		const matched = matchCountryPostcode(postcode, country)
+
+		if (matched) {
+			components.postcode = matched
+		} else {
+			countDropped(
+				tally,
+				countryPostcodeFormat(country) ? "component:postcode:not-country-format" : "component:postcode:country-has-none"
+			)
+		}
 	}
 
 	const city = clean(row.city)
@@ -200,19 +222,28 @@ export function componentsForOSMRow(
 
 	if (split) {
 		components.locality = split.locality
+	} else {
+		// The containing boundary's name stands in only where the mapper tagged no city,
+		// so a tagged value is never replaced by a position.
+		const boundary = clean(row.boundary_locality)
+
+		if (boundary) {
+			components.locality = boundary
+			countDropped(tally, "kept:locality-from-boundary")
+		}
 	}
 
-	// Dependent-locality candidates in priority order (suburb, subdistrict, district, place, city head);
-	// `district` never becomes `region`, because Vietnam's country template
-	// renders it only when no city is present.
-	const dependent = [row.suburb, row.subdistrict, row.district, row.place, split?.head]
+	// Dependent-locality candidates in priority order (suburb, subdistrict, district, place,
+	// city head, then the containing boundary's name); `district` never becomes `region`,
+	// because Vietnam's country template renders it only when no city is present.
+	const dependent = [row.suburb, row.subdistrict, row.district, row.place, split?.head, row.boundary_dependent_locality]
 		.map((value) => clean(value))
 		.find(
 			(value) =>
 				value !== null &&
 				!value.includes(",") &&
 				(street === null || !sameName(value, street)) &&
-				(split === null || !sameName(value, split.locality))
+				(!components.locality || !sameName(value, components.locality))
 		)
 
 	if (dependent) {
@@ -221,7 +252,7 @@ export function componentsForOSMRow(
 
 	const province = clean(row.province)
 
-	if (province && (!split || !sameName(province, split.locality))) {
+	if (province && (!components.locality || !sameName(province, components.locality))) {
 		components.region = province
 	}
 
@@ -317,7 +348,7 @@ export function createOSMAdapter(): CorpusAdapter {
 					countDropped(opts, "kept:country-untagged")
 				}
 
-				const components = componentsForOSMRow(row, opts)
+				const components = componentsForOSMRow(row, country, opts)
 
 				if (!components) continue
 

package/lib/adapters.ts

@@ -50,9 +50,11 @@ import { defaultAdapterRegistry } from "#adapters/registry"
 import { wofAdminAdapter } from "#adapters/wof/admin/json/adapter"
 import { wofPostalcodeAdapter } from "#adapters/wof/postalcode-json/adapter"
 import { gnafAdapter } from "#au/adapters/gnaf/adapter"
+import { azerbaijanPlacesAdapter } from "#az/adapters/opendata/adapter"
 import { brusselsAdapter } from "#be/adapters/brussels/adapter"
 import { vlaanderenAdapter } from "#be/adapters/vlaanderen/adapter"
 import { wallonieAdapter } from "#be/adapters/wallonie/adapter"
+import { gbpaAdapter } from "#bs/adapters/gbpa/adapter"
 import { czCuzkAdapter } from "#cz/adapters/cuzk/adapter"
 import { dkInspireAdapter } from "#dk/adapters/inspire/adapter"
 import { adsAdapter } from "#ee/adapters/ads/adapter"
@@ -61,11 +63,15 @@ import { esCatastroAdapter } from "#es/adapters/catastro/adapter"
 import { esGipuzkoaAdapter } from "#es/adapters/gipuzkoa/adapter"
 import { esNavarraAdapter } from "#es/adapters/navarra/adapter"
 import { ryhtiAdapter } from "#fi/adapters/ryhti/adapter"
+import { adressurFaroeAdapter } from "#fo/adapters/adressur/adapter"
 import { annuaireEducationAdapter } from "#fr/adapters/annuaire-education/adapter"
 import { banAdapter } from "#fr/adapters/ban/adapter"
 import { finessAdapter } from "#fr/adapters/finess/adapter"
 import { gbOSOpenAdapter } from "#gb/adapters/os-open/adapter"
+import { guamAddressPointsAdapter } from "#gu/adapters/bsp/adapter"
+import { isleOfManLandRegistryAdapter } from "#im/adapters/land-registry/adapter"
 import { itANACAdapter } from "#it/adapters/anac/adapter"
+import { llvLiechtensteinAdapter } from "#li/adapters/llv/adapter"
 import { bdAdressesAdapter } from "#lu/adapters/bd-adresses/adapter"
 import { varisAdapter } from "#lv/adapters/varis/adapter"
 import { nlKadasterAdapter } from "#nl/adapters/kadaster/adapter"
@@ -85,6 +91,7 @@ import { usgovImlsPlsAdapter } from "#us/adapters/usgov/imls-pls/adapter"
 import { USGovIRSBMFAdapter } from "#us/adapters/usgov/irs-bmf/adapter"
 import { usgovNADAdapter } from "#us/adapters/usgov/nad/adapter"
 import { usgovNPPESAdapter } from "#us/adapters/usgov/nppes/adapter"
+import { ideUruguayAdapter } from "#uy/adapters/ide/adapter"
 
 /**
  * Built-in adapters.
@@ -107,6 +114,13 @@ export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
 	banAdapter,
 	matrikkelenAdapter,
 	ryhtiAdapter,
+	ideUruguayAdapter,
+	llvLiechtensteinAdapter,
+	adressurFaroeAdapter,
+	isleOfManLandRegistryAdapter,
+	gbpaAdapter,
+	azerbaijanPlacesAdapter,
+	guamAddressPointsAdapter,
 	czCuzkAdapter,
 	dkInspireAdapter,
 	adsAdapter,