@mailwoman/corpus 10.5.0 → 10.7.0

package/lib/recipes/sub/venue.ts

@@ -25,6 +25,7 @@ import {
 	type IdentifierModel,
 	type LegPools,
 	mergeNamePools,
+	NameSource,
 	type PoolQuery,
 	type PromotedSurface,
 	promotedSurfacesFor,
@@ -404,7 +405,7 @@ export interface SubVenueLegStats {
 
 const LICENSE =
 	"Synthetic — OpenStreetMap venue + sub-venue names (ODbL, © OpenStreetMap contributors) and Overture Places names " +
-	"(CDLA-Permissive-2.0) over OpenAddresses / HM Land Registry Price Paid Data address skeletons"
+	"(CDLA-Permissive-2.0) over OpenAddresses / BAN / US situs / OS OpenData (OGL-UK-3.0) address skeletons"
 
 const CORPUS_VERSION = "0.16.0"
 
@@ -424,10 +425,23 @@ interface EmitContext {
 	counters: { emitted: number; skipped: number; contaminated: number }
 }
 
+/**
+ * The distinct sources of a row's pooled names, sorted, or an empty list for a row that carries none.
+ */
+function rowNameSources(...sources: Array<NameSource | null>): NameSource[] {
+	return [...new Set(sources.filter((source) => source != null))].toSorted()
+}
+
 /**
  * Renders and aligns one row.
  * Writes the row and returns whether the write succeeded.
  *
+ * The row's `name_sources` field lists the publications its pooled names were read from.
+ * The corpus schema has no such column.
+ *
+ * `jsonlToParquet` projects a row onto that schema, so the field reaches the
+ * recipe output and not a corpus parquet file.
+ *
  * The function drops rows that contain an eval-board surface or fail alignment.
  */
 function emitRow(
@@ -437,7 +451,8 @@ function emitRow(
 	groups: Group[],
 	register: Register,
 	synthMethod: string,
-	disambiguator: Record<string, string>
+	disambiguator: Record<string, string>,
+	nameSources: NameSource[]
 ): boolean {
 	const { raw, components } = renderGroups(groups, register)
 
@@ -464,7 +479,9 @@ function emitRow(
 		return false
 	}
 
-	context.write(stringifyJSON({ ...aligned.row, synth_method: synthMethod, synth_base_id: null }))
+	context.write(
+		stringifyJSON({ ...aligned.row, synth_method: synthMethod, synth_base_id: null, name_sources: nameSources })
+	)
 
 	context.counters.emitted++
 	bump(stats.byRegister, register)
@@ -503,7 +520,7 @@ function emitPositives(
 		}
 
 		const tuple = sample(pools.context, random)
-		const venue = sample(pools.venues, random)
+		const { name: venue, source: venueSource } = sample(pools.venues, random)
 
 		// Alignment cannot place two spans when one contains the other, so the loop redraws.
 		const lowVenue = venue.toLowerCase()
@@ -523,11 +540,23 @@ function emitPositives(
 		// Real mail uses both "Terminal 5, Heathrow" and "Heathrow, Terminal 5".
 		const groups = r < SUBVENUE_FIRST_CUTOFF ? [subGroup, venueGroup, ...body] : [venueGroup, subGroup, ...body]
 
-		const ok = emitRow(context, leg, stats, groups, register, `sub-venue:${form.form}`, {
-			leg: leg.locale,
-			form: form.form,
-			v: String(produced),
-		})
+		// An attested sub-venue string is an OpenStreetMap name, as `LegPools.attested` states.
+		const nameSources = rowNameSources(venueSource, form.form === "attested" ? NameSource.OpenStreetMap : null)
+
+		const ok = emitRow(
+			context,
+			leg,
+			stats,
+			groups,
+			register,
+			`sub-venue:${form.form}`,
+			{
+				leg: leg.locale,
+				form: form.form,
+				v: String(produced),
+			},
+			nameSources
+		)
 
 		if (!ok) continue
 
@@ -563,6 +592,8 @@ function emitNegatives(
 		const negativeClass = sample(available, random)
 		const register = sampleRegister(random)
 		let groups: Group[]
+		// A street negative carries no pooled name.
+		let nameSource: NameSource | null = null
 
 		if (negativeClass === NegativeClass.DesignatorStreet) {
 			groups = addressGroups(leg.country, pickTuple(streets.designator, random), true)
@@ -571,14 +602,19 @@ function emitNegatives(
 		} else if (negativeClass === NegativeClass.GateSuffixStreet) {
 			groups = addressGroups(leg.country, pickTuple(streets.gateSuffix, random), true)
 		} else {
-			const pool =
-				negativeClass === NegativeClass.RejectedVenue
-					? pools.rejectedVenues
-					: negativeClass === NegativeClass.LongerName
-						? pools.longerNames
-						: pools.unpromotedShapes
-
-			const name = sample(pool, random)
+			let name: string
+
+			if (negativeClass === NegativeClass.UnpromotedShape) {
+				name = sample(pools.unpromotedShapes, random)
+				nameSource = NameSource.OpenStreetMap
+			} else {
+				const pool = negativeClass === NegativeClass.RejectedVenue ? pools.rejectedVenues : pools.longerNames
+				const drawn = sample(pool, random)
+
+				name = drawn.name
+				nameSource = drawn.source
+			}
+
 			const tuple = sample(pools.context, random)
 
 			groups = [
@@ -591,11 +627,20 @@ function emitNegatives(
 		// This happens when no layout covers the country.
 		if (!groups.length || groups.every((group) => group.every((piece) => piece.tag === "venue"))) continue
 
-		const ok = emitRow(context, leg, stats, groups, register, `sub-venue-negative:${negativeClass}`, {
-			leg: leg.locale,
-			negative: negativeClass,
-			v: String(produced),
-		})
+		const ok = emitRow(
+			context,
+			leg,
+			stats,
+			groups,
+			register,
+			`sub-venue-negative:${negativeClass}`,
+			{
+				leg: leg.locale,
+				negative: negativeClass,
+				v: String(produced),
+			},
+			rowNameSources(nameSource)
+		)
 
 		if (!ok) continue
 
@@ -721,7 +766,7 @@ export const subVenueRecipe: CorpusRecipe = {
 		const negativeFraction = opts.negativeFraction ?? DEFAULT_NEGATIVE_FRACTION
 		const extractsDir = opts.extractsDir ?? dataRootPath("sub-venue", "extracts")
 		const venueNames = opts.venueNames ?? DEFAULT_VENUE_NAMES
-		const tuplesPath = opts.subVenueTuples ?? dataRootPath("corpus", "intermediate", "house-venue-tuples-v3.jsonl")
+		const tuplesPath = opts.subVenueTuples ?? dataRootPath("corpus", "intermediate", "house-venue-tuples-v6.jsonl")
 		const lexicon: SubVenueLexiconTable = await readSubVenueLexicon(opts.lexicon ?? defaultLexiconPath())
 
 		const context: EmitContext = {

package/lib/recipes/unit.ts

@@ -20,9 +20,14 @@ import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 import { stableSourceID } from "#adapters/source-id"
 import { readOATuples, requireRegister, type CorpusRecipe } from "#recipes/scaffold"
 import { defaultRecipeSource } from "#recipes/sources"
-import { EVAL_SOURCE, TRAIN_SOURCES, type UnitSource } from "#recipes/unit/sources"
 import { SurfaceOrigin } from "#types"
 import type { CanonicalRow } from "#types"
+import {
+	NAD_SKELETON_DIRECTORY,
+	US_SKELETON_EVAL_SOURCE,
+	US_SKELETON_TRAIN_SOURCES,
+	type USSkeletonSource,
+} from "#us/recipes/skeleton-sources"
 import { alignRow } from "#utils"
 
 /**
@@ -75,11 +80,15 @@ export interface UnitTuple {
 	region: string
 	postcode: string
 	oaUnit: string
+	/**
+	 * The register the skeleton came from, stamped on the row in place of the invocation's `--register`.
+	 */
+	register?: string
 }
 
-async function readTuples(source: UnitSource): Promise<UnitTuple[]> {
+async function readTuples(source: USSkeletonSource): Promise<UnitTuple[]> {
 	return readOATuples(source, {
-		extra: (fields, row) => ({ ...fields, region: source.region, oaUnit: row.unit ?? "" }),
+		extra: (fields, row) => ({ ...fields, region: source.region, oaUnit: row.unit ?? "", register: source.register }),
 	})
 }
 
@@ -232,7 +241,7 @@ export const unitRecipe: CorpusRecipe = {
 		const count = opts.count
 		const random = makeMulberry32(opts.seed)
 		const source = opts.sourceName ?? defaultRecipeSource("synth-unit")
-		const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES
+		const sources = opts.golden ? [US_SKELETON_EVAL_SOURCE] : US_SKELETON_TRAIN_SOURCES
 
 		const pool: UnitTuple[] = []
 
@@ -247,7 +256,10 @@ export const unitRecipe: CorpusRecipe = {
 		}
 
 		if (!pool.length) {
-			throw new Error(`No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")}?`)
+			throw new Error(
+				`No US tuples found — are the cached OA zips present in ${dataRootPath("oa-cache")} and the NAD ` +
+					`skeletons in ${dataRootPath(NAD_SKELETON_DIRECTORY)}?`
+			)
 		}
 
 		let emitted = 0
@@ -287,7 +299,8 @@ export const unitRecipe: CorpusRecipe = {
 				source,
 				source_id: stableSourceID(source, components),
 				corpus_version: "0.4.0",
-				license: "OpenAddresses US (non-VT) skeletons + injected USPS Pub-28 C2 unit designators",
+				license:
+					"OpenAddresses and National Address Database US (non-VT) skeletons + injected USPS Pub-28 C2 unit designators",
 			}
 
 			const aligned = alignRow(canonical)
@@ -303,7 +316,7 @@ export const unitRecipe: CorpusRecipe = {
 					...aligned.row,
 					recipe: "unit",
 					base_source_id: null,
-					register: requireRegister(opts, "unit"),
+					register: base.register ?? requireRegister(opts, "unit"),
 					surface: SurfaceOrigin.Composed,
 				})
 			)

package/lib/registers.ts

@@ -54,6 +54,39 @@ export const SourceRegister = {
 	 * covers Svalbard beside the mainland.
 	 */
 	MatrikkelenAdresse: "no-matrikkelen-adresse",
+	/**
+	 * `Direcciones Geográficas del Uruguay`, the national address points the Infraestructura
+	 * de Datos Espaciales publishes as one CSV per department on catalogodatos.gub.uy.
+	 */
+	UruguayIDEDirecciones: "uy-ide-direcciones",
+	/**
+	 * The `Verzeichnis der Gebäudeadressen`, Liechtenstein's official building-address index,
+	 * which the Landesverwaltung publishes on service.geo.llv.li under its ABI/OGD license.
+	 */
+	LiechtensteinGebaeudeadressen: "li-llv-gebaeudeadressen",
+	/**
+	 * `Adressur í Føroyum`, the Faroese national address register Umhvørvisstovan serves from gis.us.fo.
+	 */
+	FaroeAdressur: "fo-us-adressur",
+	/**
+	 * The Isle of Man Land Registry's `Land transactions` file, one row per registered sale,
+	 * published under the Isle of Man Open Government Licence.
+	 */
+	IsleOfManLandTransactions: "im-land-registry-transactions",
+	/**
+	 * The Grand Bahama Port Authority's Freeport address points, published on GitHub under CC BY 4.0.
+	 */
+	GrandBahamaPortAuthorityAddresses: "bs-gbpa-addresses",
+	/**
+	 * `Azərbaycanda obyektlər və məkanlar`, the objects-and-places points the Ministry
+	 * of Ecology and Natural Resources publishes on opendata.az under CC BY.
+	 */
+	AzerbaijanObjectsAndPlaces: "az-opendata-obyektler",
+	/**
+	 * The Government of Guam GISWEB Portal's `Address_Point_(Public_View)` layer,
+	 * from the Bureau of Statistics & Plans GIS.
+	 */
+	GuamAddressPoints: "gu-bsp-address-points",
 	/**
 	 * The INSPIRE Addresses theme of the Czech Office for Surveying, Mapping and Cadastre.
 	 *