@mailwoman/corpus 9.2.0 → 9.4.0

package/lib/adapters/wof-admin-json/adapter.ts

@@ -0,0 +1,231 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-admin`: Who's On First admin GeoJSON-bundle adapter.
+ *
+ *   **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at
+ *   `packages/corpus/lib/adapters/wof-admin/`, removed in this same change) was replaced by this
+ *   one because the SQLite distribution path was unworkable for the real corpus build:
+ *
+ *   1. `dist.whosonfirst.org/sqlite/` is dead (NXDOMAIN); the Geocode-Earth mirror is the only one.
+ *   2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated
+ *        as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows.
+ *   3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a
+ *        separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases
+ *        (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a
+ *        patched `is_current` predicate.
+ *
+ *   Input: a directory containing one or more cloned `whosonfirst-data-admin-<cc>` GitHub repos. Each
+ *   repo has `data/XXX/YYY/ZZZ/<wof-id>.geojson` files; `**\/*.geojson` walks the tree recursively.
+ *   Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same
+ *   record, not new records.
+ *
+ *   Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair:
+ *
+ *   - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized
+ *       variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`,
+ *       `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case:
+ *       `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become
+ *       training rows for the same WOF id.
+ *   - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2,
+ *       country → 1, county → 1.
+ *
+ *   `source_id` is `wof-admin-<wof_id>-<name-slot>-<hierarchy-variant>`. The previous SQLite adapter
+ *   used `wof-admin-<wof_id>-<hierarchy-variant>` (no name slot); the new format adds a name-slot
+ *   segment so the colloquial / preferred / per-locale variants survive dedup independently.
+ *
+ *   License: CC0. The adapter stamps every row with `CC0-1.0`.
+ */
+
+import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import {
+	COUNTRY_DISPLAY_NAME,
+	emitWOFJSONRows,
+	LOCALE_BY_COUNTRY,
+	nameSlotsFor as wofNameSlotsFor,
+	type WOFVariantSpec,
+} from "#adapters/wof-json-rows"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { buildAncestryIndex, walkFeatures, type WOFRecord } from "#utils"
+
+/**
+ * Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip.
+ *
+ * Per-adapter deliberately (the postalcode adapter carries its own): each table is a record FILTER for its adapter's
+ * emission set, not a shared vocabulary.
+ */
+function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
+	switch (placetype) {
+		case "country":
+		case "nation":
+			return "country"
+		case "macroregion":
+		case "region":
+			return "region"
+		case "macrocounty":
+		case "county":
+		case "localadmin":
+			return "subregion"
+		case "locality":
+			return "locality"
+		case "borough":
+		case "macrohood":
+		case "neighbourhood":
+		case "microhood":
+			return "dependent_locality"
+		default:
+			return undefined
+	}
+}
+
+/**
+ * Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
+ *
+ * `selfName` is the surface form to use for the record's own component (locality / region / country / subregion).
+ * Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` localized value for variant slots;
+ * ancestor names always come from the ancestor's canonical `wof:name`.
+ *
+ * Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
+ * canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
+ */
+export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
+	const selfTag = placetypeToTag(row.placetype)
+
+	if (!selfTag) return []
+
+	const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
+	const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
+	const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
+
+	const variants: WOFVariantSpec[] = []
+
+	switch (selfTag) {
+		case "locality":
+		case "dependent_locality": {
+			variants.push({ suffix: "self", components: { [selfTag]: selfName } })
+
+			if (region) {
+				variants.push({
+					suffix: "with-region",
+					components: { [selfTag]: selfName, region: region.name },
+				})
+			}
+
+			if (region && country) {
+				variants.push({
+					suffix: "with-region-country",
+					components: { [selfTag]: selfName, region: region.name, country: countryDisplay },
+				})
+			} else if (!region && country) {
+				variants.push({
+					suffix: "with-country",
+					components: { [selfTag]: selfName, country: countryDisplay },
+				})
+			}
+
+			return variants
+		}
+
+		case "region": {
+			variants.push({ suffix: "self", components: { region: selfName } })
+
+			if (country) {
+				variants.push({
+					suffix: "with-country",
+					components: { region: selfName, country: countryDisplay },
+				})
+			}
+
+			return variants
+		}
+
+		case "country": {
+			variants.push({ suffix: "self", components: { country: selfName } })
+
+			return variants
+		}
+
+		case "subregion": {
+			variants.push({ suffix: "self", components: { subregion: selfName } })
+
+			return variants
+		}
+
+		default:
+			return []
+	}
+}
+
+/**
+ * Build the per-record name-slot list. The canonical `"default"` slot uses the OpenCage-canonical country form when the
+ * record is itself a country (matches SQLite-adapter behavior); every other placetype's default slot uses `wof:name`
+ * verbatim.
+ */
+export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
+	return wofNameSlotsFor(rec, {
+		canonicalName: (record) =>
+			placetypeToTag(record.placetype) === "country"
+				? (COUNTRY_DISPLAY_NAME[record.country] ?? record.name)
+				: record.name,
+	})
+}
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const WOF_ADMIN_ADAPTER_ID = "wof-admin"
+
+/**
+ * Construct the wof-admin JSON-bundle adapter. The adapter is stateless across runs; calling this twice with the same
+ * input directory produces byte-identical `canonical.jsonl` (records are emitted in sorted `wof:id` order to be
+ * insensitive to filesystem walk ordering).
+ */
+export function createWOFAdminAdapter(): CorpusAdapter {
+	return {
+		id: WOF_ADMIN_ADAPTER_ID,
+		defaultLicense: "CC0-1.0",
+		description:
+			"Who's On First admin GeoJSON bundles (countries, regions, counties, localities) — multi-name variants per record.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			// Pass 1: scan every GeoJSON file once, build the in-memory record index.
+			// We keep only records whose placetype maps to a ComponentTag — irrelevant placetypes
+			// (campus, county-region hybrids on which Mailwoman has no opinion) are dropped here so
+			// they don't inflate the ancestry index. Country-filtered runs prune to the matching
+			// country code too; the ancestors of a same-country record live in the same admin repo.
+			const byID = new Map<number, WOFRecord>()
+
+			for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
+				if (opts.signal?.aborted) return
+
+				if (opts.country && rec.country !== opts.country) continue
+
+				if (!placetypeToTag(rec.placetype)) continue
+				byID.set(rec.id, rec)
+			}
+
+			const ancestry = buildAncestryIndex(byID)
+
+			// Pass 2: emit rows in sorted-id order for deterministic JSONL.
+			yield* emitWOFJSONRows({
+				records: byID,
+				ancestry,
+				adapterOptions: opts,
+				adapterID: WOF_ADMIN_ADAPTER_ID,
+				localeByCountry: LOCALE_BY_COUNTRY,
+				nameSlotsFor,
+				variantsFor,
+			})
+		},
+	}
+}
+
+/**
+ * Single shared instance, suitable for `defaultAdapterRegistry`.
+ */
+export const wofAdminAdapter = createWOFAdminAdapter()

package/lib/adapters/wof-json-rows.ts

@@ -0,0 +1,129 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
+ */
+
+import type { ComponentTag } from "@mailwoman/core/types"
+import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
+
+import type { AdapterOptions, CanonicalRow } from "#types"
+import { normalizeNameKey, type WOFRecord } from "#utils"
+
+/**
+ * Display name for the country, keyed by ISO 3166-1 alpha-2.
+ *
+ * Must be the **OpenCage-canonical** surface form: the `address-formatter` library expands some country names en route
+ * to its output (e.g. `"United States"` → `"United States of America"`). If `components.country` and the formatted
+ * `raw` disagree, alignment will fail downstream. Keying off the canonical form keeps the two in lockstep.
+ *
+ * Phase 1 US + FR only; extend as new locales come online. Missing countries fall back to the country row's `wof:name`,
+ * accepting the alignment risk for non-canonicalized names.
+ */
+export const COUNTRY_DISPLAY_NAME: Record<string, string> = {
+	US: "United States of America",
+	FR: "France",
+}
+
+/**
+ * BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country.
+ */
+export const LOCALE_BY_COUNTRY: Record<string, string> = {
+	US: "en-US",
+	FR: "fr-FR",
+}
+
+export interface WOFVariantSpec {
+	suffix: string
+	components: Partial<Record<ComponentTag, string>>
+}
+
+export interface NameSlotOptions {
+	/**
+	 * Canonical surface for the record's own `"default"` slot. Default `rec.name` verbatim; the admin adapter substitutes
+	 * the OpenCage-canonical {@link COUNTRY_DISPLAY_NAME} for country records.
+	 */
+	canonicalName?: (rec: WOFRecord) => string
+}
+
+/**
+ * Build the per-record name-slot list: the canonical `"default"` slot, then every `name:*` variant deduplicated against
+ * it so a redundant `"default"`-equivalent row is not emitted under a localized key.
+ */
+export function nameSlotsFor(rec: WOFRecord, options: NameSlotOptions = {}): Array<{ key: string; value: string }> {
+	const canonicalSelfName = options.canonicalName?.(rec) ?? rec.name
+
+	const seen = new Set<string>([canonicalSelfName])
+	const slots: Array<{ key: string; value: string }> = [{ key: "default", value: canonicalSelfName }]
+
+	for (const [rawKey, value] of rec.nameVariants) {
+		if (seen.has(value)) continue
+		seen.add(value)
+		slots.push({ key: normalizeNameKey(rawKey), value })
+	}
+
+	return slots
+}
+
+interface EmitWOFJSONRowsOptions {
+	records: ReadonlyMap<number, WOFRecord>
+	ancestry: ReadonlyMap<number, WOFRecord[]>
+	adapterOptions: AdapterOptions
+	adapterID: string
+	localeByCountry: Readonly<Record<string, string>>
+	shouldEmit?: (record: WOFRecord) => boolean
+	nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
+	variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
+}
+
+/**
+ * Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
+ */
+export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
+	const {
+		records,
+		ancestry,
+		adapterOptions,
+		adapterID,
+		localeByCountry,
+		shouldEmit = () => true,
+		nameSlotsFor: slotsForRecord,
+		variantsFor,
+	} = options
+
+	let emitted = 0
+
+	for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
+		if (adapterOptions.signal?.aborted) return
+		const record = records.get(id)!
+
+		if (!shouldEmit(record)) continue
+
+		for (const slot of slotsForRecord(record)) {
+			for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
+				if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
+
+				const raw = formatAddress(variant.components, record.country, { separator: ", " })
+
+				if (!raw) continue
+				const components = reconcileComponents(variant.components, raw)
+
+				if (!Object.keys(components).length) continue
+
+				yield {
+					raw,
+					components,
+					country: record.country,
+					locale: localeByCountry[record.country],
+					source: adapterID,
+					source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
+					corpus_version: "",
+					license: "CC0-1.0",
+				}
+
+				emitted++
+			}
+		}
+	}
+}

package/lib/adapters/wof-postalcode-json/adapter.ts

@@ -0,0 +1,176 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `wof-postalcode`: Who's On First postalcode GeoJSON-bundle adapter.
+ *
+ *   **Phase 1.5.1 pivot.** Replaces the previous SpatiaLite-backed implementation (formerly at
+ *   `packages/corpus/lib/adapters/wof-postalcode/`, removed in this same change). The rationale is
+ *   in `wof-admin-json/adapter.ts` and in `DECISIONS.md` — short version: the SQLite distribution
+ *   mirror is dead, the live distro tags every postcode row `mz:is_current = -1` which the old
+ *   `is_current = 1` predicate excluded, and localized `name:*` variants don't ship in the SQLite
+ *   export at all.
+ *
+ *   Input: a directory containing one or more cloned `whosonfirst-data-postalcode-<cc>` repos plus
+ *   the relevant `whosonfirst-data-admin-<cc>` repos (postcode records reference admin ancestry by
+ *   `wof:parent_id`, so the locality / region / country records must be in the same walk for the
+ *   ancestry chain to resolve). The corpus pipeline clones all four repos under
+ *   `/data/corpus/sources/wof/repos/` and points the adapter at that root.
+ *
+ *   Per live postalcode record, the adapter emits one row per `(name-variant, hierarchy-variant)`
+ *   pair:
+ *
+ *   - **Name variants**: canonical `wof:name` (slot key `default`, typically the postcode digits
+ *       themselves) plus any `name:*` variants on the postcode feature. In practice WOF postcode
+ *       records rarely carry localized name variants, so this expansion is usually a no-op — but
+ *       the code path stays symmetric with the admin adapter for consistency.
+ *   - **Hierarchy variants** (unchanged from the SQLite adapter): self, +locality, +locality+region,
+ *       +locality+region+country.
+ *
+ *   `source_id` is `wof-postalcode-<wof_id>-<name-slot>-<hierarchy-variant>`. Ancestor names always
+ *   come from the ancestor's canonical `wof:name`; this adapter does NOT iterate ancestor name
+ *   variants (e.g. it does not emit `"75008 Париж"` even when Paris has a `name:rus_x_preferred`).
+ *   That cross-product belongs to a future synthesis pass; emitting it here would multiply row
+ *   counts ~10× without a clear training-value story.
+ *
+ *   License: CC0.
+ */
+
+import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import {
+	COUNTRY_DISPLAY_NAME,
+	emitWOFJSONRows,
+	LOCALE_BY_COUNTRY,
+	nameSlotsFor as wofNameSlotsFor,
+	type WOFVariantSpec,
+} from "#adapters/wof-json-rows"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { buildAncestryIndex, walkFeatures, type WOFRecord } from "#utils"
+
+/**
+ * Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip.
+ *
+ * Per-adapter deliberately (the admin adapter carries its own): each table is a record FILTER for its adapter's
+ * emission set — this one keeps `postalcode` plus the ancestry placetypes its variants render — not a shared
+ * vocabulary.
+ */
+function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
+	switch (placetype) {
+		case "country":
+		case "nation":
+			return "country"
+		case "macroregion":
+		case "region":
+			return "region"
+		case "locality":
+			return "locality"
+		case "postalcode":
+			return "postcode"
+		default:
+			return undefined
+	}
+}
+
+/**
+ * Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
+ * the `default` slot, a `name:*` localized variant otherwise).
+ */
+export function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
+	if (placetypeToTag(row.placetype) !== "postcode") return []
+
+	const locality = ancestry.find((a) => placetypeToTag(a.placetype) === "locality")
+	const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
+	const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
+	const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
+
+	const variants: WOFVariantSpec[] = [{ suffix: "self", components: { postcode: selfName } }]
+
+	if (locality) {
+		variants.push({
+			suffix: "with-locality",
+			components: { postcode: selfName, locality: locality.name },
+		})
+	}
+
+	if (locality && region) {
+		variants.push({
+			suffix: "with-locality-region",
+			components: { postcode: selfName, locality: locality.name, region: region.name },
+		})
+	}
+
+	if (locality && region && country) {
+		variants.push({
+			suffix: "with-locality-region-country",
+			components: {
+				postcode: selfName,
+				locality: locality.name,
+				region: region.name,
+				country: countryDisplay,
+			},
+		})
+	}
+
+	return variants
+}
+
+/**
+ * Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
+ * come from `name:*` variants dedup'd against the default.
+ */
+export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
+	return wofNameSlotsFor(rec)
+}
+
+/**
+ * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
+ * it came from.
+ */
+export const WOF_POSTALCODE_ADAPTER_ID = "wof-postalcode"
+
+export function createWOFPostalcodeAdapter(): CorpusAdapter {
+	return {
+		id: WOF_POSTALCODE_ADAPTER_ID,
+		defaultLicense: "CC0-1.0",
+		description:
+			"Who's On First postalcode GeoJSON bundles (postcode → locality/region pairs). Ancestor names from sibling admin repos.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			// Pass 1: full walk. We keep every record whose placetype maps to a ComponentTag — the
+			// postcode adapter needs locality / region / country admin records in the index so it
+			// can resolve postcode ancestry, even though it only emits rows for postcode records.
+			const byID = new Map<number, WOFRecord>()
+
+			for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
+				if (opts.signal?.aborted) return
+
+				if (opts.country && rec.country !== opts.country) continue
+
+				if (!placetypeToTag(rec.placetype)) continue
+				byID.set(rec.id, rec)
+			}
+
+			const ancestry = buildAncestryIndex(byID)
+
+			// Pass 2: emit postcode rows only, sorted by id for determinism.
+			yield* emitWOFJSONRows({
+				records: byID,
+				ancestry,
+				adapterOptions: opts,
+				adapterID: WOF_POSTALCODE_ADAPTER_ID,
+				localeByCountry: LOCALE_BY_COUNTRY,
+				shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
+				nameSlotsFor,
+				variantsFor: postcodeVariantsFor,
+			})
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const wofPostalcodeAdapter = createWOFPostalcodeAdapter()