@mailwoman/corpus 10.6.0 → 10.7.0

package/lib/az/adapters/opendata/adapter.ts

@@ -0,0 +1,389 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `opendata-az`: `Azərbaycanda obyektlər və məkanlar` (objects and places in Azerbaijan), a GeoJSON of
+ * 11,807 points of interest in Baku, Sumgayit and Absheron that the Ministry of Ecology and Natural
+ * Resources publishes on opendata.az.
+ *
+ * Each point holds a place's name (`NAME`) and its address in English transliteration: `STRT_NAME` and
+ * `STRT_ADDR` for the street and number, `STATE` for the city district or settlement and `CITY` for the
+ * city. The adapter reads `NAME` as the venue, `STRT_NAME` as the street, `STRT_ADDR` as the house
+ * number, the last comma part of `STATE` (`Sabunchu,Zabrat` names the settlement Zabrat) as the dependent
+ * locality and `CITY` as the locality. Absheron is a district rather than a city, so an Absheron row
+ * takes its settlement (`Khirdalan`, `Masazir`) as the locality.
+ *
+ * `STRT_ADDR` writes `0` where the point has no number. Some street values carry the number after a
+ * comma (`Zarifa Aliyeva,38`) or a building's name in parentheses (`Azadlig Avenue(Hilton Baku)`): the
+ * number is read from the street when `STRT_ADDR` has none, and the parenthetical is removed. A block
+ * code (`558-ci0mehelle030`) is not a house number. A row with neither a number nor a venue is refused.
+ *
+ * `NAME` names a point of interest of the type `TYPE` states (`Cafe Gorush`, `School 93`, `Library by
+ * A. P. Chekhov`). No value in the 2025-08-05 file is a person's given name and surname alone: of the
+ * three that read `<Word> <surname-ending>`, each is a category word and a name (`Cafe Beyzade`,
+ * `Monument Rahimov`). A value with no run of three letters is a code and is dropped.
+ *
+ * Ninety-two points also carry their street in Azerbaijani spelling (`STRT_N_AZ`); each of those
+ * emits a second row in that spelling, with `NAME_AZ`, `STATE_AZ` and `CITY_AZ` where present
+ * (`Bakı` for `Baku`), when it renders differently from the first.
+ *
+ * The file holds no postcode. The CKAN record states `license_id: cc-by`, `Creative Commons Attribution`,
+ * with no version, so the rows carry a LicenseRef rather than an SPDX id that would assert a version.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { parseJSONStrict } from "@mailwoman/core/json"
+import { resolvePath } from "path-ts"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceIDFromParts } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import {
+	AddressRole,
+	type AdapterOptions,
+	type CanonicalRow,
+	type CorpusAdapter,
+	countDropped,
+	SurfaceOrigin,
+} from "#types"
+
+/**
+ * Registry id for this adapter, stamped into every row it emits.
+ */
+export const OPENDATA_AZ_ADAPTER_ID = "opendata-az"
+
+/**
+ * The publisher's license, Creative Commons Attribution, whose version the record leaves unstated.
+ */
+export const OPENDATA_AZ_LICENSE = "LicenseRef-CC-BY-unstated-version"
+
+/**
+ * Every jurisdiction this adapter emits, checked against a caller's `--country`.
+ */
+export const OPENDATA_AZ_COUNTRIES: readonly string[] = ["AZ"]
+
+/**
+ * The feature properties the adapter reads.
+ */
+export interface AzerbaijanPlaceRecord {
+	OBJECTID?: number | null
+	CITY?: string | null
+	STATE?: string | null
+	STRT_NAME?: string | null
+	STRT_ADDR?: string | null
+	NAME?: string | null
+	NAME_AZ?: string | null
+	STATE_AZ?: string | null
+	STRT_N_AZ?: string | null
+	CITY_AZ?: string | null
+}
+
+interface AzerbaijanFeatureCollection {
+	features?: Array<{
+		properties?: AzerbaijanPlaceRecord | null
+		geometry?: { coordinates?: readonly number[] } | null
+	}>
+}
+
+/**
+ * The spelling a row is rendered in.
+ */
+export const AzerbaijanSpelling = {
+	/**
+	 * `STRT_NAME`, `NAME`, `STATE` and `CITY`: English transliteration.
+	 */
+	Transliterated: "transliterated",
+	/**
+	 * `STRT_N_AZ`, `NAME_AZ`, `STATE_AZ` and `CITY_AZ`: Azerbaijani.
+	 */
+	Azerbaijani: "azerbaijani",
+} as const
+
+export type AzerbaijanSpelling = (typeof AzerbaijanSpelling)[keyof typeof AzerbaijanSpelling]
+
+/**
+ * A value with its whitespace collapsed, or null when the file leaves it blank (a single space).
+ */
+function present(value: string | number | null | undefined): string | null {
+	const collapsed = value == null ? "" : String(value).replaceAll(/\s+/gu, " ").trim()
+
+	return collapsed || null
+}
+
+/**
+ * A house number: digits with an optional letter, and an optional second number
+ * after a slash, backslash or hyphen (`4/66`, `23-33`, `20\116`, `8/5A`).
+ */
+const HOUSE_NUMBER = /^\d+\p{L}?(?:[/\\-]\d+\p{L}?)?$/u
+
+/**
+ * A street value that carries the house number after a comma, as `Zarifa Aliyeva,38`.
+ */
+const STREET_WITH_NUMBER = /^(.+?),\s*(\d+\p{L}?(?:\/\d+\p{L}?)?)$/u
+
+/**
+ * A block (`məhəllə`) code in the number column, as `558-ci0mehelle030`.
+ */
+const BLOCK_CODE = /mehelle/iu
+
+/**
+ * The house number `STRT_ADDR` states, with a backslash read as the slash it stands for,
+ * or null when the value is absent, `0`, or not a house number.
+ */
+function houseNumberFrom(value: string | null, tally: Pick<AdapterOptions, "dropped">): string | null {
+	if (!value || value === "0") return null
+
+	if (BLOCK_CODE.test(value)) {
+		countDropped(tally, "component:house_number:block-code")
+
+		return null
+	}
+
+	if (!HOUSE_NUMBER.test(value)) {
+		countDropped(tally, "component:house_number:not-a-number")
+
+		return null
+	}
+
+	return value.replaceAll("\\", "/")
+}
+
+/**
+ * The fields of one spelling: the street, venue, district and city columns it reads.
+ */
+interface SpellingFields {
+	street: string | null
+	venue: string | null
+	state: string | null
+	city: string | null
+}
+
+function fieldsFor(record: AzerbaijanPlaceRecord, spelling: AzerbaijanSpelling): SpellingFields {
+	if (spelling === AzerbaijanSpelling.Azerbaijani) {
+		return {
+			street: present(record.STRT_N_AZ),
+			venue: present(record.NAME_AZ),
+			state: present(record.STATE_AZ) ?? present(record.STATE),
+			city: present(record.CITY_AZ) ?? present(record.CITY),
+		}
+	}
+
+	return {
+		street: present(record.STRT_NAME),
+		venue: present(record.NAME),
+		state: present(record.STATE),
+		city: present(record.CITY),
+	}
+}
+
+/**
+ * Whether a city value is the Absheron district, in either spelling.
+ */
+function isAbsheron(city: string): boolean {
+	return /^Ab(?:sh|ş)eron$/iu.test(city)
+}
+
+/**
+ * Maps one record onto the components the corpus asserts in one spelling,
+ * or returns null when the row is refused.
+ *
+ * Each refusal and each dropped component increments its reason in `dropped`.
+ */
+export function componentsForAzerbaijanRecord(
+	record: AzerbaijanPlaceRecord,
+	dropped?: Pick<AdapterOptions, "dropped">,
+	spelling: AzerbaijanSpelling = AzerbaijanSpelling.Transliterated
+): CanonicalRow["components"] | null {
+	const tally = dropped ?? {}
+	const fields = fieldsFor(record, spelling)
+	let street = fields.street
+
+	if (!street) {
+		countDropped(tally, "row:no-street")
+
+		return null
+	}
+
+	if (/[{@]/u.test(street)) {
+		countDropped(tally, "row:street-markup")
+
+		return null
+	}
+
+	let houseNumber = houseNumberFrom(present(record.STRT_ADDR), tally)
+	const carried = STREET_WITH_NUMBER.exec(street)
+
+	if (carried) {
+		street = carried[1]!.trim()
+		houseNumber ??= carried[2]!
+	}
+
+	if (/\(/u.test(street)) {
+		street = street.replaceAll(/\s*\([^)]*\)\s*/gu, " ").trim()
+		countDropped(tally, "component:street:parenthetical-removed")
+	}
+
+	if (!street || street.includes(",")) {
+		countDropped(tally, "row:street-holds-several-parts")
+
+		return null
+	}
+
+	const components: CanonicalRow["components"] = { street }
+
+	if (houseNumber) {
+		components.house_number = houseNumber
+	}
+
+	const venue = fields.venue
+
+	// A value with no run of three letters (`555`, `SM`, `T T`) is a code rather than a name.
+	if (venue && !/\p{L}{3}/u.test(venue)) {
+		countDropped(tally, "component:venue:code")
+	} else if (venue) {
+		components.venue = venue
+	}
+
+	if (!components.house_number && !components.venue) {
+		countDropped(tally, "row:no-house-number-or-venue")
+
+		return null
+	}
+
+	const city = fields.city
+	const district = fields.state?.split(",").pop()?.trim() || null
+
+	if (!city) {
+		countDropped(tally, "row:no-city")
+
+		return null
+	}
+
+	if (isAbsheron(city)) {
+		components.locality = district ?? city
+	} else {
+		components.locality = city
+
+		if (district && district.toLowerCase() !== city.toLowerCase() && !/^Sum[gq]ayit$/iu.test(district)) {
+			components.dependent_locality = district
+		}
+	}
+
+	return components
+}
+
+export function createAzerbaijanPlacesAdapter(): CorpusAdapter {
+	return {
+		id: OPENDATA_AZ_ADAPTER_ID,
+		defaultLicense: OPENDATA_AZ_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.AzerbaijanObjectsAndPlaces,
+		surface: SurfaceOrigin.Rendered,
+		description:
+			"Azərbaycanda obyektlər və məkanlar (Ministry of Ecology and Natural Resources, opendata.az): addressed places in Baku, Sumgayit and Absheron.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !OPENDATA_AZ_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(OPENDATA_AZ_ADAPTER_ID, OPENDATA_AZ_COUNTRIES, opts.country)
+			}
+
+			// The publication is one 5 MB file, so it is parsed whole.
+			const path = resolvePath(opts.inputPath)
+			const collection = parseJSONStrict<AzerbaijanFeatureCollection>(await readLocalTextFile(path))
+
+			if (!Array.isArray(collection.features)) {
+				throw new TypeError(`${OPENDATA_AZ_ADAPTER_ID} adapter: ${path} holds no GeoJSON features array`)
+			}
+
+			let emitted = 0
+
+			for (const feature of collection.features) {
+				const record = feature.properties
+
+				if (!record) {
+					countDropped(opts, "row:no-properties")
+
+					continue
+				}
+
+				const [lon, lat] = feature.geometry?.coordinates ?? []
+				let transliterated: string | null = null
+
+				const spellings = present(record.STRT_N_AZ)
+					? [AzerbaijanSpelling.Transliterated, AzerbaijanSpelling.Azerbaijani]
+					: [AzerbaijanSpelling.Transliterated]
+
+				for (const spelling of spellings) {
+					if (opts.signal?.aborted) return
+
+					if (opts.limit !== undefined && emitted >= opts.limit) return
+
+					// The Azerbaijani row is counted under its own prefix, so its refusals
+					// do not read as refusals of records.
+					const tally = spelling === AzerbaijanSpelling.Azerbaijani ? { dropped: new Map<string, number>() } : opts
+					const components = componentsForAzerbaijanRecord(record, tally, spelling)
+
+					if (spelling === AzerbaijanSpelling.Azerbaijani) {
+						for (const [reason, count] of tally.dropped ?? []) {
+							const key = `azerbaijani-spelling:${reason}`
+
+							opts.dropped?.set(key, (opts.dropped.get(key) ?? 0) + count)
+						}
+					}
+
+					if (!components) continue
+
+					const rendered = formatAddressRow(components, "AZ", { singleLine: true })
+
+					if (!rendered) {
+						countDropped(opts, "row:render-failed")
+
+						continue
+					}
+
+					for (const tag of rendered.unplaced) {
+						countDropped(opts, `component:${tag}:not-rendered`)
+					}
+
+					const { raw, components: aligned } = rendered
+
+					if (spelling === AzerbaijanSpelling.Transliterated) {
+						transliterated = raw
+					} else if (raw === transliterated) {
+						countDropped(opts, "azerbaijani-spelling:row:renders-as-transliterated")
+
+						continue
+					}
+
+					yield {
+						raw,
+						components: aligned,
+						country: "AZ",
+						locale: spelling === AzerbaijanSpelling.Azerbaijani ? "az-AZ" : "en-AZ",
+						source: OPENDATA_AZ_ADAPTER_ID,
+						// `OBJECTID` is unique per feature.
+						// The spelling separates a feature's two rows.
+						source_id: stableSourceIDFromParts(OPENDATA_AZ_ADAPTER_ID, {
+							...aligned,
+							objectid: present(record.OBJECTID) ?? "",
+							spelling,
+							lat: typeof lat === "number" ? String(lat) : "",
+							lon: typeof lon === "number" ? String(lon) : "",
+						}),
+						corpus_version: "",
+						license: OPENDATA_AZ_LICENSE,
+					}
+
+					emitted++
+				}
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const azerbaijanPlacesAdapter = createAzerbaijanPlacesAdapter()

package/lib/bs/adapters/gbpa/adapter.ts

@@ -0,0 +1,248 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * `gbpa-bs`: the address points of the Grand Bahama Port Authority (GBPA), covering the City of Freeport
+ * and Lucaya on Grand Bahama, which the GBPA publishes in its `GBPA/gis-datafiles` repository on GitHub.
+ * The adapter reads the CSV export, `CSV/addresses.csv`.
+ *
+ * A Freeport address number is a block and a parcel joined by a hyphen (`8-29 Seagull Drive`), and either
+ * part may be lettered (`UU-3`, `13-43C`, `W-A-10`). The adapter reads `AddressNum` as the house number,
+ * `FullStName` as the street, `City` as the locality, `Island` as the region and `AltName` as a venue
+ * (`Lucayan International School`, `Shoreline`). The export truncates some parcel ids to five characters and
+ * a star (`Minis*`), and those numbers are dropped.
+ *
+ * The export repeats the street's name in a few records (`Rippindale Rippindale`, and across an embedded
+ * line break `Santa Maria \r\nSanta Maria Avenue`). The adapter collapses the repeat and counts it.
+ *
+ * The GBPA licenses the data under CC BY 4.0 and asks for the statement "Data Provided by The Grand Bahama
+ * Port Authority". Every row records that license.
+ */
+
+import { formatAddressRow } from "@mailwoman/codex/address/format"
+import { readLocalBuffer } from "@mailwoman/core/fs/readers"
+import { resolvePath } from "path-ts"
+import { CSVSpliterator } from "spliterator"
+
+import { UnsupportedCountryError } from "#adapters/errors"
+import { stableSourceIDFromParts } from "#adapters/source-id"
+import { SourceRegister } from "#registers"
+import {
+	AddressRole,
+	type AdapterOptions,
+	type CanonicalRow,
+	type CorpusAdapter,
+	countDropped,
+	SurfaceOrigin,
+} from "#types"
+
+/**
+ * Registry id for this adapter, stamped into every row it emits.
+ */
+export const GBPA_BS_ADAPTER_ID = "gbpa-bs"
+
+/**
+ * The license the GBPA publishes the repository's data under.
+ */
+export const GBPA_BS_LICENSE = "CC-BY-4.0"
+
+/**
+ * The attribution statement the GBPA asks for.
+ */
+export const GBPA_BS_ATTRIBUTION = "Data Provided by The Grand Bahama Port Authority"
+
+/**
+ * Every jurisdiction this adapter emits, checked against a caller's `--country`.
+ */
+export const GBPA_BS_COUNTRIES: readonly string[] = ["BS"]
+
+/**
+ * The columns the adapter reads from `CSV/addresses.csv`.
+ */
+export interface GBPAAddressRecord {
+	Y: string
+	X: string
+	AddID: string
+	AltName: string
+	Suffix: string
+	FullStName: string
+	AddressNum: string
+	City: string
+	Island: string
+}
+
+/**
+ * A value with its whitespace, including the export's embedded line breaks, collapsed, or null when empty.
+ */
+function present(value: string | undefined): string | null {
+	const trimmed = value?.replaceAll(/\s+/gu, " ").trim() ?? ""
+
+	return trimmed || null
+}
+
+/**
+ * A block and parcel number: alphanumeric parts joined by hyphens (`8-29`, `UU-3`, `W-A-10`, `G-3B10`, `L`).
+ */
+const ADDRESS_NUMBER = /^[A-Z\d]+(?:-[A-Z\d]+)*$/iu
+
+/**
+ * A street's name repeated before its type (`Santa Maria Santa Maria Avenue`)
+ * or after it (`Wragby Wragby`).
+ */
+const REPEATED_NAME = /^(.+?) \1(?= |$)/u
+
+/**
+ * A street value that is only a type word, with no name before it.
+ */
+const TYPE_WORD_ONLY = /^(?:road|lane|drive|avenue|street|court|close|way|place|circle|terrace|crescent)$/iu
+
+/**
+ * Initials and a surname alone (`J.S. Johnson`) name a person rather than a place or a business.
+ */
+const INITIALS_AND_SURNAME = /^(?:\p{Lu}\.\s*)+\p{Lu}\p{Ll}+$/u
+
+/**
+ * Maps one record onto the components the corpus asserts, or returns null when the row is refused.
+ *
+ * Each refusal and each dropped or repaired component increments its reason in `dropped`.
+ */
+export function componentsForGBPARecord(
+	record: GBPAAddressRecord,
+	dropped?: Pick<AdapterOptions, "dropped">
+): CanonicalRow["components"] | null {
+	const tally = dropped ?? {}
+	let street = present(record.FullStName)
+
+	if (!street) {
+		countDropped(tally, "row:no-street")
+
+		return null
+	}
+
+	if (REPEATED_NAME.test(street)) {
+		street = street.replace(REPEATED_NAME, "$1")
+		countDropped(tally, "component:street:repeated-name-collapsed")
+	}
+
+	if (TYPE_WORD_ONLY.test(street)) {
+		countDropped(tally, "row:street-is-a-type-word")
+
+		return null
+	}
+
+	const components: CanonicalRow["components"] = { street }
+	const number = present(record.AddressNum)
+
+	if (number && ADDRESS_NUMBER.test(number)) {
+		components.house_number = number
+	} else if (number?.includes("*")) {
+		countDropped(tally, "component:house_number:truncated")
+	} else if (number) {
+		countDropped(tally, "component:house_number:not-a-block-parcel-number")
+	}
+
+	const venue = present(record.AltName)
+
+	if (venue && !/\p{L}{3}/u.test(venue)) {
+		countDropped(tally, "component:venue:code")
+	} else if (venue && /(\p{L})\1{3,}$/u.test(venue)) {
+		// `Bellevue Gifts & Suppliesccccc` carries a run of repeated keystrokes.
+		countDropped(tally, "component:venue:malformed")
+	} else if (venue && INITIALS_AND_SURNAME.test(venue)) {
+		countDropped(tally, "component:venue:names-a-person")
+	} else if (venue) {
+		components.venue = venue
+	}
+
+	if (!components.house_number && !components.venue) {
+		countDropped(tally, "row:no-house-number-or-name")
+
+		return null
+	}
+
+	const locality = present(record.City)
+	const region = present(record.Island)
+
+	if (locality) {
+		components.locality = locality
+	}
+
+	if (region) {
+		components.region = region
+	}
+
+	if (!components.locality) {
+		countDropped(tally, "row:no-locality")
+
+		return null
+	}
+
+	return components
+}
+
+export function createGBPAAdapter(): CorpusAdapter {
+	return {
+		id: GBPA_BS_ADAPTER_ID,
+		defaultLicense: GBPA_BS_LICENSE,
+		addressRole: AddressRole.Premise,
+		register: SourceRegister.GrandBahamaPortAuthorityAddresses,
+		surface: SurfaceOrigin.Rendered,
+		description: "Grand Bahama Port Authority address points (Freeport and Lucaya), GBPA/gis-datafiles CSV export.",
+
+		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
+			if (opts.country && !GBPA_BS_COUNTRIES.includes(opts.country)) {
+				throw new UnsupportedCountryError(GBPA_BS_ADAPTER_ID, GBPA_BS_COUNTRIES, opts.country)
+			}
+
+			// The file is 8 MB, so it is decoded whole.
+			// The BOM would otherwise prefix the first header.
+			const text = (await readLocalBuffer(resolvePath(opts.inputPath))).toString("utf8").replace(/^/u, "")
+			const records = CSVSpliterator.from<GBPAAddressRecord>(Buffer.from(text, "utf8"), { normalizeKeys: false })
+			let emitted = 0
+
+			for (const record of records) {
+				if (opts.signal?.aborted) return
+
+				if (opts.limit !== undefined && emitted >= opts.limit) return
+
+				const components = componentsForGBPARecord(record, opts)
+
+				if (!components) continue
+
+				const rendered = formatAddressRow(components, "BS", { singleLine: true })
+
+				if (!rendered) {
+					countDropped(opts, "row:render-failed")
+
+					continue
+				}
+
+				for (const tag of rendered.unplaced) {
+					countDropped(opts, `component:${tag}:not-rendered`)
+				}
+
+				const { raw, components: aligned } = rendered
+
+				yield {
+					raw,
+					components: aligned,
+					country: "BS",
+					locale: "en-BS",
+					source: GBPA_BS_ADAPTER_ID,
+					// The GBPA's own address id separates two points the rendered components cannot.
+					source_id: stableSourceIDFromParts(GBPA_BS_ADAPTER_ID, { ...aligned, add_id: record.AddID ?? "" }),
+					corpus_version: "",
+					license: GBPA_BS_LICENSE,
+				}
+
+				emitted++
+			}
+		},
+	}
+}
+
+/**
+ * The configured adapter instance registered with the corpus builder.
+ */
+export const gbpaAdapter = createGBPAAdapter()

package/lib/build/types.ts

@@ -13,6 +13,7 @@ import type { PathBuilderLike } from "path-ts"
 
 import type { ParquetManifest } from "#parquet/writers"
 import type { AdapterRunManifest } from "#runner"
+import type { SourceRightsRecord } from "#source-register/corpus-inputs"
 import type { AdapterOptions, CorpusAdapter } from "#types"
 import type { LicensePolicy, LicenseRefusalKind } from "#utils/license"
 import type { SplitManifest } from "#utils/split"
@@ -142,6 +143,16 @@ export interface BuildCorpusOptions {
 	 * @defaultValue {@linkcode BuildProfile.Exploratory}
 	 */
 	profile?: BuildProfile
+
+	/**
+	 * Why a local experiment admits sources whose register rows lack an elected
+	 * decision permitting `transform` and `train`.
+	 *
+	 * Without it the build refuses such a source before any adapter runs.
+	 * With it the build logs every gap and records the corpus as not shippable
+	 * in {@linkcode BuildCorpusManifest.source_rights}.
+	 */
+	unshippableReason?: string
 }
 
 /**
@@ -231,4 +242,9 @@ export interface BuildCorpusManifest {
 	 * That file is the frozen record of which sources contributed and under which terms.
 	 */
 	training_manifest_digest: string
+	/**
+	 * Whether every source this build emitted reads only register sources whose decisions permit
+	 * `transform` and `train`, and the gaps an `unshippableReason` admitted when they do not.
+	 */
+	source_rights: SourceRightsRecord
 }