@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/adapters/dedup-key.ts

@@ -0,0 +1,40 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The key that decides two rows describe the same address.
+ */
+
+import type { ComponentTag } from "@mailwoman/codex/component"
+
+import type { CanonicalRow } from "#types"
+
+/**
+ * Canonical dedup key for a row.
+ *
+ * Two rows that share this key are duplicates and the first one wins.
+ * The key combines `country` with the sorted `components` dictionary and the row's
+ * `raw` text, lower-cased with whitespace collapsed.
+ *
+ * License and provenance fields stay out of the key, so two adapters emitting
+ * the same address produce the same key.
+ * That alone does not deduplicate the corpus across adapters: `runAdapter` holds its key store as a local
+ * and discards it when the adapter returns, so each adapter is deduplicated only against itself.
+ *
+ * A cross-adapter pass would hold every adapter's keys at once and would have to choose
+ * which adapter's copy of an address survives.
+ * That choice sets the row's `source`, `license` and `register`, which makes it a
+ * corpus-level decision the runner does not take today.
+ *
+ * An augmented row is never deduplicated against the row it was fanned from, because
+ * `recipe.recipe` joins the key when present and each augmentation variant therefore survives.
+ */
+export function canonicalDedupKey(row: CanonicalRow): string {
+	const sortedKeys = Object.keys(row.components).toSorted() as ComponentTag[]
+	const compPart = sortedKeys.map((k) => `${k}=${row.components[k] ?? ""}`).join("\u001F")
+	const rawNorm = row.raw.toLowerCase().replaceAll(/\s+/g, " ").trim()
+	const recipePart = row.recipe ? `\u001E${row.recipe.recipe}` : ""
+
+	return `${row.country}\u001E${rawNorm}\u001E${compPart}${recipePart}`
+}

package/lib/adapters/errors.ts

@@ -0,0 +1,21 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   The errors an adapter raises for a caller's own input.
+ */
+
+/**
+ * Raised when a caller asks for a country outside the set the adapter's source publishes.
+ *
+ * An adapter whose source spans several jurisdictions checks `opts.country` against its own set,
+ * so a caller reads which jurisdictions the source publishes instead of receiving zero rows.
+ */
+export class UnsupportedCountryError extends Error {
+	constructor(adapterID: string, covered: readonly string[], requested: string) {
+		super(`${adapterID} adapter: the dataset covers ${covered.join(", ")}, got country=${requested}`)
+
+		this.name = "UnsupportedCountryError"
+	}
+}

package/lib/adapters/geonames/adapter.ts

@@ -3,79 +3,65 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `geonames`: GeoNames populated-places consumer (https://www.geonames.org/, CC-BY-4.0).
+ *   Reads populated places from a GeoNames country dump and emits locality rows.
  *
- *   GeoNames is a global gazetteer of ~12M features. This adapter ingests the POPULATED PLACES
- *   (`feature_class = "P"`, excluding historical/abandoned/destroyed variants) from a per-country
- *   dump file — global locality coverage, including the small towns and villages a coarser admin
- *   gazetteer (WOF) lacks. It's the cheapest path to broadening the corpus's LOCALE coverage.
- *
- *   Input: a per-country tab-separated dump (e.g. `US.txt` from
- *   `https://download.geonames.org/export/dump/`, 19 columns, no header). Two sibling files in the
- *   same directory supply human-readable names (downloaded once from the same place):
- *
- *   - `admin1CodesASCII.txt` — `<CC>.<admin1_code>` → region name (e.g. `US.VT` → "Vermont").
- *   - `countryInfo.txt` — ISO alpha-2 → country name (e.g. `US` → "United States"); `#`-commented. If a
- *       sibling is missing, the corresponding component is simply omitted (graceful degradation).
- *
- *   Output: per place, up to two hierarchy variants (mirroring `wof-admin`'s with/without-country
- *   balance so the model sees both domestic and international order) —
- *
- *   1. `{ locality, region }` → "City, Region"
- *   2. `{ locality, region, country }` → "City, Region, Country" `componentsPresentIn` drops any
- *        component that didn't survive into the rendered `raw`.
- *
- *   License: stamped `"CC-BY-4.0"` per row (GeoNames' terms); provenance is the `geonames-<id>` key.
+ *   Region and country names come from `admin1CodesASCII.txt` and `countryInfo.txt` in the same directory. When a
+ *   file is missing, rows omit that component.
  */
 
-import { componentsPresentIn } from "@mailwoman/codex/address-format"
+import { componentsPresentIn } from "@mailwoman/codex/address/format"
+import { readUnquotedTSV } from "@mailwoman/core/fs/delimited"
 import { pathExists } from "@mailwoman/core/fs/readers"
-import { dirname, join } from "path-ts"
-import { TSVSpliterator } from "spliterator"
+import { PathBuilder } from "path-ts"
 
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
 
 /**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
+ * The source ID on rows from this adapter.
  */
 export const GEONAMES_ADAPTER_ID = "geonames"
 /**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
+ * The license on rows from this adapter.
  */
 export const GEONAMES_DEFAULT_LICENSE = "CC-BY-4.0"
 
 /**
- * GeoNames main-table column indices (0-based; see the export README).
+ * Zero-based column indices in a GeoNames main-table dump.
  */
-const COL = {
+export const GEONAMES_MAIN_COLUMNS = {
 	geonameid: 0,
 	name: 1,
+	asciiname: 2,
 	alternatenames: 3,
+	latitude: 4,
+	longitude: 5,
 	featureClass: 6,
 	featureCode: 7,
 	country: 8,
 	admin1: 10,
+	population: 14,
 } as const
 
+const COL = GEONAMES_MAIN_COLUMNS
+
 /**
- * Populated-place feature codes that are NOT current real places — skip them.
+ * The adapter skips these feature codes for historical, abandoned, or destroyed populated places.
  */
 const NON_CURRENT_PPL = new Set(["PPLH", "PPLQ", "PPLW", "PPLCH"])
 
 /**
- * Load `admin1CodesASCII.txt` → Map("<CC>.<admin1>" → region name). Empty map if absent.
+ * Loads region names keyed by `<CC>.<admin1>`.
+ * Returns an empty map when the file is missing.
  */
-async function loadAdmin1(dir: string): Promise<Map<string, string>> {
+async function loadAdmin1(dir: PathBuilder): Promise<Map<string, string>> {
 	const map = new Map<string, string>()
-	const fp = join(dir, "admin1CodesASCII.txt")
+	const fp = dir("admin1CodesASCII.txt")
 
 	if (!(await pathExists(fp))) return map
 
-	// `header: false` — the file is headerless, and the spliterator eats row 1 as a header otherwise.
-	for await (const cols of TSVSpliterator.fromAsync(fp, { header: false })) {
+	for await (const cols of readUnquotedTSV(fp)) {
 		if (cols[0] && cols[1]) {
 			map.set(cols[0], cols[1])
 		}
@@ -85,20 +71,19 @@ async function loadAdmin1(dir: string): Promise<Map<string, string>> {
 }
 
 /**
- * Load `countryInfo.txt` → Map(ISO → country name). Empty map if absent. The file is `#`-commented.
+ * Loads country names keyed by ISO code.
+ * Returns an empty map when the file is missing.
  */
-async function loadCountries(dir: string): Promise<Map<string, string>> {
+async function loadCountries(dir: PathBuilder): Promise<Map<string, string>> {
 	const map = new Map<string, string>()
-	const fp = join(dir, "countryInfo.txt")
+	const fp = dir("countryInfo.txt")
 
 	if (!(await pathExists(fp))) return map
 
-	// `header: false` — the file's header IS a `#` comment, so it falls out with the other comments
-	// rather than being consumed as column names.
-	for await (const cols of TSVSpliterator.fromAsync(fp, { header: false })) {
+	for await (const cols of readUnquotedTSV(fp)) {
 		if (cols[0]?.startsWith("#")) continue
 
-		// ISO(0), ISO3(1), iso-numeric(2), fips(3), Country(4), ...
+		// Column 0 holds the ISO code and column 4 the country name.
 		if (cols[0] && cols[4]) {
 			map.set(cols[0], cols[4])
 		}
@@ -107,20 +92,25 @@ async function loadCountries(dir: string): Promise<Map<string, string>> {
 	return map
 }
 
+/**
+ * Creates the GeoNames populated-place adapter.
+ */
 export function createGeonamesAdapter(): CorpusAdapter {
 	return {
 		id: GEONAMES_ADAPTER_ID,
 		defaultLicense: GEONAMES_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.GeoNames,
+		surface: SurfaceOrigin.Rendered,
 		description:
 			"GeoNames populated places (CC-BY-4.0) — global locality coverage incl. small towns, with region/country names from the sibling admin1/countryInfo files.",
 
 		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			const dir = dirname(opts.inputPath)
+			const dir = PathBuilder.from(opts.inputPath).dirname()
 			const admin1 = await loadAdmin1(dir)
 			const countries = await loadCountries(dir)
 
-			// `header: false` — the per-country dump is headerless.
-			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
+			const rows = readUnquotedTSV(opts.inputPath)
 
 			let emitted = 0
 
@@ -146,8 +136,8 @@ export function createGeonamesAdapter(): CorpusAdapter {
 				const region = admin1.get(`${cc}.${(rec[COL.admin1] ?? "").trim()}`)
 				const country = countries.get(cc)
 
-				// Two hierarchy variants (domestic + international order) — but only emit the
-				// distinct ones the available names support.
+				// With a region, the adapter emits locality-region and locality-region-country rows.
+				// Without one, it emits a single row with the country when known.
 				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = []
 
 				if (region) {
@@ -194,6 +184,6 @@ export function createGeonamesAdapter(): CorpusAdapter {
 }
 
 /**
- * The configured adapter instance registered with the corpus builder.
+ * The adapter instance registered with the corpus builder.
  */
 export const geonamesAdapter = createGeonamesAdapter()

package/lib/adapters/geonames/postal/adapter.ts

@@ -3,66 +3,61 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `geonames-postal`: GeoNames postal-code dump consumer (https://www.geonames.org/, CC-BY-4.0).
- *
- *   The GeoNames postal export (`https://download.geonames.org/export/zip/<CC>.zip`) is a clean,
- *   per-country `postcode → place → admin1` table with the place + region NAMES inline (no aux-file
- *   join needed). It broadens the corpus's postcode→locality→region coverage to ~80 countries, well
- *   beyond `wof-postalcode`/the coordinate-first table — forward coverage for the multi-locale
- *   goal.
- *
- *   Input: a per-country postal dump (`<CC>.txt`, 12 tab-separated columns, no header): country,
- *   postcode, place, admin1_name, admin1_code, admin2__, admin3__, lat, lon, accuracy.
- *
- *   Output: per row, postcode-FIRST (international) variants — the common order for the non-US
- *   locales this fills (US postcodes are already covered by TIGER/WOF, which use postcode-LAST):
- *
- *   1. `{ postcode, locality }` → "AD100 Canillo"
- *   2. `{ postcode, locality, region }` → "AD100 Canillo, Canillo" Prefer configuring this adapter for
- *        non-US countries; for US, the postcode-last sources are the right order. License:
- *        `"CC-BY-4.0"` per row (attribute "GeoNames").
+ *   Reads a per-country GeoNames postal dump and emits postcode-first locality rows. Each dump row includes its place and
+ *   region names inline.
  */
 
-import { componentsPresentIn } from "@mailwoman/codex/address-format"
-import { TSVSpliterator } from "spliterator"
+import { componentsPresentIn } from "@mailwoman/codex/address/format"
+import { readUnquotedTSV } from "@mailwoman/core/fs/delimited"
 
-import { stableSourceID } from "#adapters/utils"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
+import { stableSourceID } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import { AddressRole, type AdapterOptions, type CanonicalRow, type CorpusAdapter, SurfaceOrigin } from "#types"
 
 /**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
+ * The source ID on rows from this adapter.
  */
 export const GEONAMES_POSTAL_ADAPTER_ID = "geonames-postal"
 /**
- * License carried by this source (CC-BY-4.0), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
+ * The license on rows from this adapter.
  */
 export const GEONAMES_POSTAL_DEFAULT_LICENSE = "CC-BY-4.0"
 
 /**
- * GeoNames postal-dump columns (0-based): country, postcode, place, admin1_name, admin1_code, admin2_name, ….
+ * Zero-based column indices in a GeoNames postal dump.
  *
- * Shared with `tools/postcode-triples.ts`, which additionally reads `admin2Name` — the CITY for the IN/MX/PT-shaped
- * exports whose `place` column is a street or colonia.
+ * The coordinates locate the postcode.
+ * That position may differ from the locality's position.
  */
-export const GEONAMES_POSTAL_COLUMNS = { country: 0, postcode: 1, place: 2, admin1Name: 3, admin2Name: 5 } as const
+export const GEONAMES_POSTAL_COLUMNS = {
+	country: 0,
+	postcode: 1,
+	place: 2,
+	admin1Name: 3,
+	admin2Name: 5,
+	latitude: 9,
+	longitude: 10,
+} as const
 
+/**
+ * Creates the GeoNames postal adapter.
+ */
 export function createGeonamesPostalAdapter(): CorpusAdapter {
 	return {
 		id: GEONAMES_POSTAL_ADAPTER_ID,
 		defaultLicense: GEONAMES_POSTAL_DEFAULT_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.GeoNamesPostal,
+		surface: SurfaceOrigin.Rendered,
 		description:
 			"GeoNames postcodes (CC-BY-4.0) — multi-locale postcode→locality→region, names inline; international postcode-first order.",
 
 		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			// `header: false` — the GeoNames postal dump is headerless, and the spliterator would
-			// otherwise consume row 1 as column names and lose its first postcode.
-			const rows = TSVSpliterator.fromAsync(opts.inputPath, { header: false })
+			const rows = readUnquotedTSV(opts.inputPath)
 
 			let emitted = 0
 
-			for await (const rec of rows as AsyncIterable<string[]>) {
+			for await (const rec of rows) {
 				if (opts.signal?.aborted) break
 
 				if (opts.limit !== undefined && emitted >= opts.limit) break
@@ -79,8 +74,7 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
 				if (!postcode || !locality) continue
 				const region = (rec[GEONAMES_POSTAL_COLUMNS.admin1Name] ?? "").trim()
 
-				// Postcode-first (international) variants. Skip the region variant when admin1 just
-				// repeats the place (common for city-states / micro-admin) to avoid "X X" noise.
+				// The region row is skipped when the region name equals the locality name.
 				const variants: Array<{ slot: string; comp: CanonicalRow["components"]; raw: string }> = [
 					{ slot: "pl", comp: { postcode, locality }, raw: `${postcode} ${locality}` },
 				]
@@ -117,6 +111,6 @@ export function createGeonamesPostalAdapter(): CorpusAdapter {
 }
 
 /**
- * The configured adapter instance registered with the corpus builder.
+ * The adapter instance registered with the corpus builder.
  */
 export const geonamesPostalAdapter = createGeonamesPostalAdapter()