@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/adapters/libpostal-dictionary.ts

@@ -0,0 +1,39 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The curated libpostal dictionary reader the street-decompose modules share.
+ */
+
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { resourceDictionaryPath } from "@mailwoman/core/paths"
+import { TextSpliterator } from "spliterator"
+
+/**
+ * Loads one curated libpostal dictionary as a lower-cased form set.
+ *
+ * A dictionary line reads `canonical|abbr|abbr|…` and every form is indexed,
+ * so a lookup matches the canonical spelling and each abbreviation alike.
+ * A line opening with `#` is a comment.
+ *
+ * `resourceDictionaryPath` resolves both layouts, `core/data/…` from source and the packaged `out/` tree.
+ * The candidate list this replaced named one of them twice, guessed a third path from `process.cwd()`,
+ * and swallowed every error while probing, so a corrupt dictionary reported as a missing one.
+ *
+ * The largest dictionary is 8.4 KB, and each caller loads it once per process at module load.
+ */
+export async function loadLibpostalDictionary(language: string, filename: string): Promise<Set<string>> {
+	const text = await readLocalTextFile(resourceDictionaryPath("libpostal", language, filename))
+	const set = new Set<string>()
+
+	for (const line of TextSpliterator.from(text)) {
+		if (line.startsWith("#")) continue
+
+		for (const form of TextSpliterator.from(line, { delimiter: "|" })) {
+			set.add(form.toLowerCase())
+		}
+	}
+
+	return set
+}

package/lib/adapters/openaddresses/adapter.ts

@@ -3,61 +3,40 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
+ * `openaddresses`: line-delimited GeoJSON adapter for openaddresses.io exports.
  *
- *   OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
- *   (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
- *   line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
- *   matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
- *
- *   The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
- *   open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
- *   per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
- *   propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
- *   or re-attribute by license at training time.
- *
- *   Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
- *   but the row-level data doesn't include a country code, so the adapter refuses to run without
- *   one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
- *   inputs JSON.
- *
- *   Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
- *   variants are accepted because legacy dumps used UPPERCASE):
- *
- *   | Property | ComponentTag | | ------------- |
- *   -------------------------------------------------------------- | | `number` | `house_number` |
- *   | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
- *   `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
- *   per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
- *   then synthesize)|
- *
- *   `district` is intentionally NOT mapped — for US data it carries borough or county and would
- *   inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
- *   for non-US locales where district names DO appear on the envelope.
+ * Country must be explicit because OpenAddresses organizes files by country while the row data
+ * has no country code. A per-row `LICENSE` property wins where present and the configured
+ * `defaultLicense` covers the remaining rows. `district` stays unmapped because US data uses it for a borough or county.
+ * The adapter omits that field because mapping it would inflate alignment quarantine.
  */
 
+import { formatAddressRow } from "@mailwoman/codex/address/format"
 import { tryParsingJSON } from "@mailwoman/core/json"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
 import { TextSpliterator } from "spliterator"
 
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-import { SHARE_ALIKE_PATTERN } from "#utils"
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
+import { createLicenseVerdictCache, LicensePolicy } from "#utils"
 
 /**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
+ * Registry id for this adapter.
+ *
+ * Stamped into every row it emits, so a corpus record can be traced back to the dataset it came from.
  */
 export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
 /**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
+ * License assigned to each row from this source (CC-BY-4.0), so downstream consumers
+ * inherit the terms rather than having to look them up.
  */
 export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
 
 /**
- * Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
- * this interface documents the canonical lowercase form after normalization.
+ * Subset of OpenAddresses Feature properties the adapter inspects.
+ *
+ * The runtime accepts uppercase or lowercase keys.
+ * This interface documents the canonical lowercase form after normalization.
  */
 interface OaProperties {
 	hash?: string
@@ -91,7 +70,8 @@ function normalizeProperties(raw: unknown): OaProperties {
 }
 
 /**
- * Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes.
+ * Parse a single ND-GeoJSON line.
+ * Return null for blanks, comments, or non-Feature shapes.
  */
 function parseFeatureLine(line: string): OaProperties | null {
 	const trimmed = line.trim()
@@ -109,31 +89,50 @@ function parseFeatureLine(line: string): OaProperties | null {
 
 export interface OpenaddressesAdapterOptions {
 	/**
-	 * Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
-	 * license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
+	 * Per-row license used when a Feature lacks an explicit `LICENSE` property.
+	 *
+	 * Defaults to `CC-BY-4.0` — the most common license across the OpenAddresses collection.
+	 * Override per dump via the runner's adapter-options passthrough.
 	 */
 	defaultLicense?: string
 
 	/**
-	 * Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
-	 * act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
-	 * exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
-	 * `--exclude-share-alike` is the normal path.
+	 * Whether this adapter emits a row whose per-file license requires share-alike.
+	 *
+	 * The default is `true`.
+	 * `buildCorpus({ licensePolicy })` and `mw corpus build --license-policy share-alike-free`
+	 * express a refusal at build level.
+	 *
+	 * The policy reads the obligations of every adapter's rows under one policy and records what it refused.
+	 *
+	 * Pass `false` only for an adapter-scoped drop, such as a fixture that must use one license.
 	 */
 	allowShareAlike?: boolean
 }
 
 /**
- * Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
- * known to carry a single license throughout (e.g. a PDDL-only state slice).
+ * Build an OpenAddresses adapter.
+ *
+ * The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
+ * known to use a single license throughout (e.g. a PDDL-only state extract).
  */
 export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
 	const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
 	const allowShareAlike = opts.allowShareAlike ?? true
+	// The same reading the build applies, so an adapter-scoped drop
+	// and a build-level policy refuse the same rows.
+	// OpenAddresses stamps a per-file license.
+	// A file's value can be prose naming a share-alike register rather than an identifier.
+	const shareAlike = createLicenseVerdictCache(LicensePolicy.ShareAlikeFree)
 
 	return {
 		id: OPENADDRESSES_ADAPTER_ID,
 		defaultLicense,
+		addressRole: AddressRole.Premise,
+		// OpenAddresses redistributes national and municipal registers with terms that differ per file.
+		// A row's source file identifies its upstream, rather than this adapter.
+		register: SourceRegister.OpenAddresses,
+		surface: SurfaceOrigin.Rendered,
 		description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
 
 		async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
@@ -145,9 +144,7 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
 
 			const country = adapterOpts.country
 
-			// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
-			// malformed lines by returning null); passing the path string lets the lib own + dispose
-			// the file handle, including on an early `break`.
+			// The path string lets the library own and dispose the file handle, including on an early `break`.
 			const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
 
 			let emitted = 0
@@ -170,15 +167,14 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
 					const region = props.region?.trim() ?? ""
 					const postcode = props.postcode?.trim() ?? ""
 
-					// A row is only useful if it has, at minimum, a street + (postcode OR locality).
-					// Pure point-only rows would land in quarantine anyway.
+					// A row needs a street plus a postcode or locality, or the aligner quarantines it.
 					if (!street) continue
 
 					if (!city && !postcode) continue
 
 					const license = (props.license?.trim() || defaultLicense).trim()
 
-					if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
+					if (!allowShareAlike && shareAlike.read(license).refusal !== null) {
 						shareAlikeBlocked++
 
 						continue
@@ -210,14 +206,11 @@ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {
 						components.postcode = postcode
 					}
 
-					const raw = formatAddress(components, country, { separator: ", " })
-
-					if (!raw) continue
-
-					const aligned = reconcileComponents(components, raw)
+					const rendered = formatAddressRow(components, country, { singleLine: true })
 
-					if (!Object.keys(aligned).length) continue
+					if (!rendered) continue
 
+					const { raw, components: aligned } = rendered
 					const sourceIDSeed = props.hash?.trim() || props.id?.trim()
 
 					const sourceID = sourceIDSeed

package/lib/adapters/osm/adapter.ts

@@ -0,0 +1,356 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `osm`: OpenStreetMap address adapter for the countries no permissive source covers.
+ *
+ * Every row lists `ODbL-1.0`, whose recorded obligations include share-alike. A build run under
+ * `LicensePolicy.ShareAlikeFree` refuses these rows at ingest, so they reach only the open weights.
+ * That is why `defaultLicense` is not an option on this adapter.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { extractDelimited } from "@mailwoman/core/scripting/arguments"
+import { stripCombiningMarks } from "@mailwoman/normalize/fold"
+import { TextSpliterator } from "spliterator"
+
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import {
+	AddressRole,
+	type AdapterOptions,
+	type CanonicalRow,
+	type CorpusAdapter,
+	countDropped,
+	SurfaceOrigin,
+} from "#types"
+
+/**
+ * Registry id for this adapter, stamped into every row it emits.
+ */
+export const OSM_ADAPTER_ID = "osm"
+
+/**
+ * OpenStreetMap's license.
+ *
+ * Its recorded obligations include share-alike, so a build run under
+ * `LicensePolicy.ShareAlikeFree` refuses every row carrying it.
+ */
+export const OSM_LICENSE = "ODbL-1.0"
+
+/**
+ * The URL of the ODbL license text.
+ */
+export const OSM_LICENSE_URL = "https://opendatacommons.org/licenses/odbl/1-0/"
+
+/**
+ * The attribution OSM requires in redistributed data and derived works.
+ *
+ * A database built from OSM embeds it, and a model card carries it when OSM-derived
+ * rows reached the checkpoint, which the effective training manifest records.
+ */
+export const OSM_ATTRIBUTION =
+	"© OpenStreetMap contributors. Data licensed under the Open Database License (ODbL) 1.0 " +
+	`(${OSM_LICENSE_URL}); see https://www.openstreetmap.org/copyright.`
+
+/**
+ * The per-row shape `@mailwoman/osm`'s `emit-corpus-jsonl` writes.
+ */
+interface OSMCorpusRow {
+	street?: string
+	number?: string
+	postcode?: string
+	suburb?: string
+	city?: string
+	unit?: string
+	place?: string
+	subdistrict?: string
+	district?: string
+	province?: string
+	country?: string
+}
+
+const MAX_STREET_WORDS = 8
+
+/**
+ * Whether `addr:housenumber` holds a street designator — digits with an optional letter,
+ * fraction or dash suffix (`12`, `188a`, `14/E`, `1/1146`, `167-c`) or a one-
+ * or two-letter block prefix (`B-77`, `L58`, `R 948`) — rather than free text such
+ * as `House 34, Road 4, Sector 9`, `Plot #27` or a name.
+ */
+export function housenumberIsDesignator(value: string): boolean {
+	const trimmed = value.trim()
+
+	return /^\d+[A-Za-z]?(?:[/-]\d*[A-Za-z]?)?$/u.test(trimmed) || /^[A-Za-z]{1,2}[-\s]?\d+[A-Za-z]?$/u.test(trimmed)
+}
+
+/**
+ * Whether `addr:street` matches the street-name shape.
+ *
+ * The value has no comma and contains at most {@link MAX_STREET_WORDS} words.
+ * Direction phrases such as `Near Cozy Water Park`, `Opposite Askari Towers` or `Behind …` return false.
+ */
+export function isStreetName(value: string): boolean {
+	const trimmed = value.trim()
+
+	return (
+		trimmed.length > 0 &&
+		!trimmed.includes(",") &&
+		trimmed.split(/\s+/u).length <= MAX_STREET_WORDS &&
+		!/^(?:near|opp(?:osite)?|behind|beside|next to|adjacent)\b/iu.test(trimmed)
+	)
+}
+
+/**
+ * Split an `addr:city` carrying a neighborhood ahead of the city (`Mirpur 10, Dhaka` →
+ * locality `Dhaka`, head `Mirpur 10`); a value without a comma is only the locality.
+ */
+export function splitCityValue(value: string): { locality: string; head: string | null } {
+	const parts = extractDelimited(value)
+
+	if (parts.length < 2) {
+		return { locality: value.trim(), head: null }
+	}
+
+	return { locality: parts.at(-1)!, head: parts.slice(0, -1).join(", ") }
+}
+
+const PLACEHOLDERS = new Set(["n/a", "na", "none", "-", "nil"])
+
+function clean(value: string | undefined): string | null {
+	const trimmed = value?.trim() ?? ""
+
+	return trimmed && !PLACEHOLDERS.has(trimmed.toLowerCase()) ? trimmed : null
+}
+
+function parseLine(line: string): OSMCorpusRow | null {
+	const trimmed = line.trim()
+
+	if (!trimmed || trimmed.startsWith("#")) return null
+
+	const parsed = tryParsingJSON(trimmed)
+
+	return parsed && typeof parsed === "object" ? (parsed as OSMCorpusRow) : null
+}
+
+/**
+ * Maps one jsonl row onto the components the corpus asserts, or returns null when the row is refused.
+ *
+ * A row with a street-shaped `addr:street` is a street address.
+ * A row with no `addr:street` at all is admitted as a streetless premise when its house
+ * number is designator-shaped and a tag that maps to `locality` or `dependent_locality`
+ * (`addr:city`, `addr:place`, `addr:suburb`, `addr:subdistrict`, `addr:district`)
+ * carries the identity the street would have carried.
+ *
+ * That is OSM's documented scheme for addresses numbered within a named place
+ * rather than along a street, and it is the `streetless-premise-identity` shape.
+ * A row whose `addr:street` holds a line rather than a name stays refused:
+ * the mapper supplied a street, and the value is a line.
+ *
+ * Each refusal and each discarded component increments its reason in `dropped`.
+ */
+export function componentsForOSMRow(
+	row: OSMCorpusRow,
+	dropped?: Pick<AdapterOptions, "dropped">
+): CanonicalRow["components"] | null {
+	const tally = dropped ?? {}
+	const street = clean(row.street)
+	const number = clean(row.number)
+	const designator = number !== null && housenumberIsDesignator(number)
+
+	if (street && !isStreetName(street)) {
+		countDropped(tally, "row:street-not-a-name")
+
+		return null
+	}
+
+	if (!street && !designator) {
+		countDropped(tally, number ? "row:streetless-house-number-not-designator" : "row:streetless-no-house-number")
+
+		return null
+	}
+
+	const components: CanonicalRow["components"] = street ? { street } : {}
+
+	if (designator) {
+		components.house_number = number
+	} else if (number) {
+		countDropped(tally, "component:house_number:not-designator")
+	}
+
+	const unit = clean(row.unit)
+
+	if (unit) {
+		components.unit = unit
+	}
+
+	const postcode = clean(row.postcode)
+
+	if (postcode && /^\d{4,6}$/u.test(postcode)) {
+		components.postcode = postcode
+	} else if (postcode) {
+		countDropped(tally, "component:postcode:not-4-to-6-digits")
+	}
+
+	const city = clean(row.city)
+	const split = city ? splitCityValue(city) : null
+
+	if (split) {
+		components.locality = split.locality
+	}
+
+	// Dependent-locality candidates in priority order (suburb, subdistrict, district, place, city head);
+	// `district` never becomes `region`, because Vietnam's country template
+	// renders it only when no city is present.
+	const dependent = [row.suburb, row.subdistrict, row.district, row.place, split?.head]
+		.map((value) => clean(value ?? undefined))
+		.find(
+			(value) =>
+				value !== null &&
+				!value.includes(",") &&
+				(street === null || !sameName(value, street)) &&
+				(split === null || !sameName(value, split.locality))
+		)
+
+	if (dependent) {
+		components.dependent_locality = dependent
+	}
+
+	const province = clean(row.province)
+
+	if (province && (split === null || !sameName(province, split.locality))) {
+		components.region = province
+	}
+
+	// A streetless premise is a house number within a named place, so it needs a locality
+	// or a dependent locality, from any of the tags that map to them.
+	if (!street && !components.locality && !components.dependent_locality) {
+		countDropped(tally, "row:streetless-no-locality")
+
+		return null
+	}
+
+	// An address row needs another component alongside the street or the streetless house number.
+	// Coarse adapters already teach bare names.
+	if (Object.keys(components).length === 1) {
+		countDropped(tally, street ? "row:street-only" : "row:house-number-only")
+
+		return null
+	}
+
+	return components
+}
+
+/**
+ * Vietnamese admin-generic prefixes stripped before name comparison, so a province that
+ * repeats the city with a prefix (`Thành phố Hà Nội`) is read as the repeat it is.
+ */
+const NAME_PREFIXES = /^(?:thanh pho|tinh|tp\.?|quan|phuong|huyen|thi xa)\s+/u
+
+/**
+ * Whether two place names are the same name: case, whitespace, diacritics and a leading
+ * admin generic folded (`Bắc Ninh` = `Bac Ninh`; `Hà Nội` = `Thành phố Hà Nội`).
+ */
+export function sameName(a: string, b: string): boolean {
+	const fold = (value: string): string =>
+		stripCombiningMarks(value)
+			.replaceAll("đ", "d")
+			.replaceAll("Đ", "D")
+			.toLowerCase()
+			.replaceAll(/\s+/gu, " ")
+			.trim()
+			.replace(NAME_PREFIXES, "")
+
+	return fold(a) === fold(b)
+}
+
+export function createOSMAdapter(): CorpusAdapter {
+	return {
+		id: OSM_ADAPTER_ID,
+		defaultLicense: OSM_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.OpenStreetMap,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"OpenStreetMap addresses (ODbL, share-alike): per-country JSONL from a Geofabrik extract, for the countries no permissive source covers.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (!opts.country) {
+				throw new Error("osm adapter: --country is required (the JSONL is per-country and rows omit a country field)")
+			}
+
+			const country = opts.country
+			let emitted = 0
+
+			for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
+				if (opts.signal?.aborted) break
+
+				if (opts.limit !== undefined && emitted >= opts.limit) break
+
+				const row = parseLine(line)
+
+				if (!row) {
+					if (line.trim() && !line.trim().startsWith("#")) {
+						countDropped(opts, "row:unparseable-line")
+					}
+
+					continue
+				}
+
+				// A Geofabrik extract's polygon extends past the border, so its records
+				// include addresses in the neighboring country.
+				// A record whose own `addr:country` names another country is refused.
+				// A record without the tag is kept and counted, so the size of the remaining
+				// uncertainty is in the manifest rather than assumed to be zero.
+				const tagged = clean(row.country)?.toUpperCase()
+
+				if (tagged && tagged !== country) {
+					countDropped(opts, "row:country-tag-mismatch")
+
+					continue
+				}
+
+				if (!tagged) {
+					countDropped(opts, "kept:country-untagged")
+				}
+
+				const components = componentsForOSMRow(row, opts)
+
+				if (!components) continue
+
+				const rendered = formatAddressRow(components, country, { singleLine: true })
+
+				if (!rendered) {
+					countDropped(opts, "row:render-failed")
+
+					continue
+				}
+
+				const { raw, components: aligned } = rendered
+
+				for (const tag of rendered.unplaced) {
+					countDropped(opts, `component:${tag}:not-rendered`)
+				}
+
+				yield {
+					raw,
+					components: aligned,
+					country,
+					source: OSM_ADAPTER_ID,
+					source_id: stableSourceID(OSM_ADAPTER_ID, aligned),
+					corpus_version: "",
+					license: OSM_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const osmAdapter = createOSMAdapter()