@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/subvenue/table.ts

@@ -0,0 +1,264 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * @file The sub-venue lexicon's record schema and the shipped vocabulary it is seeded from.
+ *
+ * The seeds mirror `neural/venue-structure.ts`. `@mailwoman/corpus` has no dependency on `@mailwoman/neural`,
+ * so this file declares the shipped vocabulary locally. `sub-venue-lexicon.test.ts` pins the seed values
+ * and fails when either copy changes.
+ */
+
+import type { SubVenuePromotion } from "#subvenue/promotions"
+
+/**
+ * This table's own data version.
+ *
+ * Bump when the source vintages or the build semantics change.
+ */
+export const SUBVENUE_LEXICON_VERSION = "0.2.0"
+
+/**
+ * Which side of the containment relation a designator names.
+ *
+ * It mirrors `@mailwoman/osm/sdk`'s `SubVenueTier`, re-declared for the same
+ * dependency-direction reason as the seed.
+ */
+export const LexiconTier = {
+	SubVenue: "subvenue",
+	Venue: "venue",
+} as const
+
+export type LexiconTier = (typeof LexiconTier)[keyof typeof LexiconTier]
+
+/**
+ * One designator record — a venue-interior (or containing-venue) structural noun.
+ */
+export interface SubVenueDesignator {
+	/**
+	 * Canonical id, lowercase English.
+	 *
+	 * It matches `neural/venue-structure.ts`'s `VENUE_STRUCTURE_DESIGNATORS` wherever the two overlap.
+	 */
+	id: string
+	tier: LexiconTier
+	/**
+	 * Whether this designator may be preceded by a {@link SubVenueModifier} — the `North Terminal` shape;
+	 * `gate` and `building` are excluded because `<modifier> <id>` is also an ordinary street
+	 * name ("East Gate"), so admitting them turns a correct street parse into a sub-venue one.
+	 */
+	modifierEligible: boolean
+	/**
+	 * Whether the shipped span proposer recognizes this designator; `false` means no consumer reads it yet.
+	 */
+	shipped: boolean
+	/**
+	 * Where the term comes from, one entry per attesting source (`wof:placetype`, `osm:aeroway=terminal`,
+	 * `wikidata:Q849706`, `overture:airport_terminal`), sorted so a regenerate is stable.
+	 */
+	provenance: string[]
+}
+
+/**
+ * One positional modifier — the `North`/`Upper`/`Main` half of `North Terminal`.
+ */
+export interface SubVenueModifier {
+	id: string
+	shipped: boolean
+	provenance: string[]
+}
+
+/**
+ * One surface form stores a phrase.
+ *
+ * It records the corresponding entity and where the source attested it.
+ */
+export interface SubVenueSurface {
+	/**
+	 * The phrase, lowercased for Latin script and left as written otherwise.
+	 *
+	 * The Turkish `İ` (U+0130) is `\p{Script=Latin}` and folds to `i` plus a combining dot above, so it
+	 * round-trips through a form its own locale would not write (no shipped code depends on that today).
+	 */
+	phrase: string
+	recordID: string
+	recordKind: "designator" | "modifier"
+	/**
+	 * BCP-47-ish language subtag as the source wrote it (`en`, `ja`, `zh-Hant`, `pt-BR`),
+	 * or `und` when the source gave an untagged default name.
+	 */
+	lang: string
+	/**
+	 * ISO 3166-1 alpha-2 of the data the phrase was attested in, `""` for vocabulary
+	 * sources that attest a term's existence rather than its use.
+	 *
+	 * This is the axis promotion is decided on (`hall` is attested 3,274 times in `GB`), never a global census.
+	 */
+	region: string
+	/**
+	 * `wikidata:label`, `wikidata:alt`, `osm:name`, `osm:name:<lang>`, `overture:name`,
+	 * `derived:head-noun`, or `seed`.
+	 */
+	source: string
+	/**
+	 * Whether a human has approved this surface for parsing use in its region.
+	 *
+	 * Machine-derived entries start `false`; only a matching {@link SubVenuePromotion}
+	 * flips it, so a consumer that gates a parse must filter on this.
+	 */
+	curated: boolean
+	/**
+	 * How many source features attested this exact phrase when the source counts (OSM, Overture);
+	 * Vocabulary sources attest phrase existence and use a count of `0`.
+	 * OSM and Overture sources count occurrences.
+	 */
+	observations: number
+	/**
+	 * The rule-assigned designator of the features that had this phrase, with a count each
+	 * (`platform:3205 campus:49` for GB's `hall`); empty for vocabulary sources.
+	 *
+	 * Without it an `observations` count is a magnitude with no sign (`hall` on a `platform` row is a
+	 * British bus stop called after a village hall. On a `terminal` row it is a real German departure hall).
+	 */
+	context: Record<string, number>
+}
+
+/**
+ * The measured shape of a designator's identifier half — what follows `Gate`/`Terminal` in real data.
+ *
+ * Derived from OSM `ref` values.
+ * Each of Berlin's 26 `aeroway=gate` features has a `ref` and no name.
+ *
+ * The extractor renders `Gate A12` from `<designator> <ref>`.
+ * Generation requires the identifier distribution.
+ */
+export interface IdentifierShape {
+	designatorID: string
+	/**
+	 * ISO 3166-1 alpha-2 of the extract this distribution was measured in.
+	 *
+	 * It is per-region because the shapes differ (GB gates are 70% bare digits, Japanese platform
+	 * refs a different range), so a recipe for a French address should sample France's distribution.
+	 */
+	region: string
+	/**
+	 * A coarse class: `digit` (`5`), `letter` (`B`), `letter-digit` (`A12`),
+	 * `digit-letter` (`2F`), `range` (`16-18`, `0/1`), or `other`.
+	 */
+	shape: string
+	observations: number
+	/**
+	 * Up to eight real values, sorted.
+	 */
+	examples: string[]
+}
+
+/**
+ * One input source's provenance, copied off its fetch manifest.
+ */
+export interface SubVenueLexiconSource {
+	id: string
+	origin: string
+	license: string
+	retrieved: string
+	rows: number
+}
+
+/**
+ * The committed table.
+ */
+export interface SubVenueLexiconTable {
+	version: string
+	sources: SubVenueLexiconSource[]
+	designators: SubVenueDesignator[]
+	modifiers: SubVenueModifier[]
+	surfaces: SubVenueSurface[]
+	identifierShapes: IdentifierShape[]
+	/**
+	 * Every curation decision taken against this table, promotion and rejection,
+	 * each with the census that backs it.
+	 *
+	 * A rejection is as required as a promotion because it stops the next reader
+	 * re-proposing `hall` for en-GB.
+	 */
+	promotions: SubVenuePromotion[]
+}
+
+/**
+ * The vocabulary that already ships in `neural/venue-structure.ts`, re-declared
+ * below for the module docstring's dependency-direction reason.
+ *
+ * `tier` is added here: the seven WOF placetypes plus `terminal`/`gate` are all venue-interior,
+ * while `campus` and `building` name a whole venue as often as a part of one
+ * but are marked `subvenue` because that is the role the span proposer uses them in.
+ */
+export const SHIPPED_DESIGNATOR_SEED: ReadonlyArray<{
+	id: string
+	modifierEligible: boolean
+	provenance: string[]
+}> = [
+	{ id: "arcade", modifierEligible: true, provenance: ["wof:placetype"] },
+	{ id: "building", modifierEligible: false, provenance: ["wof:placetype"] },
+	{ id: "campus", modifierEligible: true, provenance: ["wof:placetype"] },
+	{ id: "concourse", modifierEligible: true, provenance: ["wof:placetype"] },
+	{ id: "enclosure", modifierEligible: false, provenance: ["wof:placetype"] },
+	{ id: "gate", modifierEligible: false, provenance: ["osm:aeroway=gate"] },
+	{ id: "installation", modifierEligible: false, provenance: ["wof:placetype"] },
+	{ id: "terminal", modifierEligible: true, provenance: ["osm:aeroway=terminal"] },
+	{ id: "wing", modifierEligible: true, provenance: ["wof:placetype"] },
+]
+
+/**
+ * The shipped positional modifiers, re-declared from `neural/venue-structure.ts`'s `VENUE_STRUCTURE_MODIFIERS`.
+ */
+export const SHIPPED_MODIFIER_SEED: readonly string[] = [
+	"central",
+	"east",
+	"front",
+	"inner",
+	"lower",
+	"main",
+	"north",
+	"outer",
+	"rear",
+	"south",
+	"upper",
+	"west",
+]
+
+/**
+ * Designators the lexicon adds beyond what ships, each with the source that attests it.
+ *
+ * None is `modifierEligible` because that claim requires a confound board for each term and locale.
+ * A promotion marks a surface usable while preserving the modifier grammar.
+ */
+export const PROPOSED_DESIGNATORS: ReadonlyArray<{
+	id: string
+	tier: LexiconTier
+	provenance: string[]
+}> = [
+	{ id: "airport", tier: LexiconTier.Venue, provenance: ["osm:aeroway=aerodrome"] },
+	{ id: "hall", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q240854"] },
+	{ id: "pier", tier: LexiconTier.SubVenue, provenance: ["overture:pier"] },
+	{ id: "platform", tier: LexiconTier.SubVenue, provenance: ["osm:public_transport=platform", "osm:railway=platform"] },
+	{ id: "satellite", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q15990706"] },
+	{ id: "station", tier: LexiconTier.Venue, provenance: ["osm:railway=station"] },
+]
+
+/**
+ * `designatorID` → Wikidata QID, matching `fetch/wikidata-subvenue.ts`'s `SUBVENUE_CONCEPTS`.
+ *
+ * The builder remains a pure function over parsed input.
+ * The test compares both maps.
+ */
+export const CONCEPT_QIDS: Readonly<Record<string, string>> = {
+	terminal: "Q849706",
+	gate: "Q247739",
+	concourse: "Q862212",
+	campus: "Q209465",
+	building: "Q41176",
+	arcade: "Q186637",
+	hall: "Q240854",
+	satellite: "Q15990706",
+}

package/lib/surfaces/locale.ts

@@ -0,0 +1,246 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Render real address tuples into locale-specific surface strings.
+ *
+ *   Public address registers supply inputs such as OA, HM Land Registry and CNIG.
+ *   LINZ-derived extracts also supply inputs. The module applies OpenCage templates to
+ *   choose ordering and punctuation by country.
+ *
+ *   Keep the emitted source ids stable: they are persisted in built corpora and referenced by
+ *   training configs. `recipes/sources.ts` records each one under its operation spelling
+ *   (`rendered-de`) and the `synth-*` spelling it retired on 2026-09-26.
+ */
+
+import { formatAddress } from "@mailwoman/codex/address/format"
+
+import type { CanonicalRow } from "#types"
+
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+	weights. We keep them inline so the output distribution is easy to read. */
+
+/**
+ * Real address tuple (for example one OpenAddresses row).
+ *
+ * Street and locality are required.
+ * Other tuple fields are optional.
+ */
+export interface LocaleBaseTuple {
+	house_number?: string
+	street: string
+	locality: string
+	/**
+	 * Sub-locality below locality (for example suburb/district).
+	 * When present, it is rendered between street and locality.
+	 */
+	dependent_locality?: string
+	region?: string
+	postcode?: string
+}
+
+/**
+ * @deprecated Alias — use LocaleBaseTuple.
+ */
+export type GermanBaseTuple = LocaleBaseTuple
+
+export interface RenderedLocaleRow {
+	raw: string
+	components: CanonicalRow["components"]
+	locale: string
+}
+
+/**
+ * @deprecated Alias — use RenderedLocaleRow.
+ */
+export type RenderedGermanRow = RenderedLocaleRow
+
+export interface LocaleRenderOpts {
+	random?: () => number
+	/**
+	 * The order used to render the same components.
+	 *
+	 * `"native"` (default) uses the country's own template (DE → house-after-street, postcode-before-city).
+	 * `"international"` renders house-first and postcode-after-city.
+	 */
+	order?: "native" | "international"
+	/**
+	 * Postcode surface shape.
+	 *
+	 * `"conventional"` (default) canonicalizes to the country's rendered form
+	 * (NL: OA's glued `1011AB` → the spaced `1011 AB`).
+	 * `"as-source"` keeps the source's own surface.
+	 *
+	 * Today, only NL differs between these modes.
+	 */
+	postcodeShape?: "conventional" | "as-source"
+	/**
+	 * How the native-order render joins street and house number.
+	 *
+	 * The OpenCage ES template comma-joins (`Calle Mayor, 12`, the official Spanish convention).
+	 * OA-derived feeds and our ES eval space-join (`calle mayor 12`, the observed form on all 3,000 eval rows).
+	 *
+	 * `"template"` (default) keeps the template's own join.
+	 * `"space"` collapses `<street>, <house_number>` → `<street> <house_number>` after rendering.
+	 * International order ignores this.
+	 */
+	nativeHouseJoin?: "template" | "space"
+	/**
+	 * The string between rendered address lines.
+	 *
+	 * `", "` (default) is the template's own join.
+	 * `" "` renders the comma-free single-line register for dictation or a copy out of a
+	 * one-field form, as in `Neusser Str. 12 Nippes 50733 Köln` for the same components.
+	 *
+	 * Only native order uses this option.
+	 */
+	separator?: ", " | " "
+}
+
+/**
+ * @deprecated Alias — use LocaleRenderOpts.
+ */
+export type GermanRenderOpts = LocaleRenderOpts
+
+/**
+ * ISO-3166 alpha-2 → BCP-47 tag for the emitted rows (primary language per country).
+ */
+const LOCALE_TAG: Record<string, string> = {
+	DE: "de-DE",
+	ES: "es-ES",
+	IT: "it-IT",
+	NL: "nl-NL",
+	GB: "en-GB",
+	FR: "fr-FR",
+	US: "en-US",
+	NZ: "en-NZ",
+}
+
+/**
+ * Canonicalize a postcode to the form the country's template renders,
+ * so the stored component aligns verbatim against `raw`.
+ *
+ * NL is the main case: OA may store `1011AB`, while the NL template emits `1011 AB`.
+ * Other countries pass through unchanged.
+ */
+function normalizePostcode(postcode: string, country: string): string {
+	if (country === "NL") {
+		const m = /^(\d{4})\s*([A-Za-z]{2})$/.exec(postcode)
+
+		if (m) return `${m[1]} ${m[2]!.toUpperCase()}`
+	}
+
+	return postcode
+}
+
+/**
+ * True when `value` appears verbatim and as a standalone token.
+ */
+function tokenPresent(raw: string, value: string): boolean {
+	if (!raw.includes(value)) return false
+	// Reject numeric substring collisions (for example house "2" inside postcode "12623").
+	const i = raw.indexOf(value)
+	const before = raw[i - 1]
+	const after = raw[i + value.length]
+	const isDigit = (c: string | undefined) => c !== undefined && c >= "0" && c <= "9"
+
+	if (/^\d+$/.test(value) && (isDigit(before) || isDigit(after))) return false
+
+	return true
+}
+
+/**
+ * Render one tuple into a locale-ordered `{raw, components}` row via OpenCage templates,
+ * with light variation (house number and postcode may be dropped).
+ *
+ * Returns `null` when the tuple is too thin or a component would not align cleanly.
+ *
+ * Region handling depends on order:
+ * - native omits region for alignment reliability
+ * - international includes region in the tail
+ *
+ * Pass `opts.order: "international"` to render the same components house-first and postcode-after-city
+ * instead (see {@link LocaleRenderOpts.order}), matching common feed-style layouts.
+ */
+export function renderLocaleRow(
+	base: LocaleBaseTuple,
+	country: string,
+	opts: LocaleRenderOpts = {}
+): RenderedLocaleRow | null {
+	const random = opts.random ?? Math.random
+	const order = opts.order ?? "native"
+
+	if (!base.street || !base.locality) return null
+
+	const components: CanonicalRow["components"] = { street: base.street, locality: base.locality }
+
+	// Sub-locality is part of the address body in both orders.
+	// If the template does not surface it verbatim, the row is dropped below.
+	if (base.dependent_locality) {
+		components.dependent_locality = base.dependent_locality
+	}
+
+	// The sampler keeps the house number in about 80% of rows.
+	// The remaining rows use street-only forms.
+	if (base.house_number && random() < 0.8) {
+		components.house_number = base.house_number
+	}
+
+	// ~85% keep postcode.
+	// Canonicalization uses the country's rendered form.
+	// `postcodeShape: "as-source"` is applied after rendering.
+	if (base.postcode && random() < 0.85) {
+		components.postcode = normalizePostcode(base.postcode, country)
+	}
+
+	// International order places region in the tail.
+	// Native order omits it for alignment.
+	if (order === "international" && base.region) {
+		components.region = base.region
+	}
+
+	// Native order uses the country's template.
+	// International order uses the US template.
+	// No random draw here, keeping RNG sequence stable.
+	const renderCountry = order === "international" ? "US" : country
+	const separator = order === "native" ? (opts.separator ?? ", ") : ", "
+	let raw = formatAddress(components, renderCountry, { separator })
+
+	if (!raw) return null
+
+	// Optional native-order rewrite: `<street>, <hn>` -> `<street> <hn>`.
+	if (order === "native" && separator === ", " && opts.nativeHouseJoin === "space" && components.house_number) {
+		raw = raw.replace(
+			`${components.street}, ${components.house_number}`,
+			`${components.street} ${components.house_number}`
+		)
+	}
+
+	// `postcodeShape: "as-source"`: rewrite both raw and component to source form.
+	// Done post-render because templates may normalize the postcode.
+	if (opts.postcodeShape === "as-source" && components.postcode && base.postcode) {
+		const sourceForm = base.postcode.trim()
+
+		if (sourceForm && sourceForm !== components.postcode && raw.includes(components.postcode)) {
+			raw = raw.replace(components.postcode, sourceForm)
+			components.postcode = sourceForm
+		}
+	}
+
+	// The row is dropped if any component is absent from the rendered address.
+	for (const value of Object.values(components)) {
+		if (!value || !tokenPresent(raw, value)) return null
+	}
+
+	return { raw, components, locale: LOCALE_TAG[country] ?? country.toLowerCase() }
+}
+
+/**
+ * German wrapper over {@link renderLocaleRow}.
+ *
+ * Kept for the `german` recipe (`de/recipes/locale.ts`) and tests.
+ */
+export function renderGermanRow(base: LocaleBaseTuple, opts: LocaleRenderOpts = {}): RenderedLocaleRow | null {
+	return renderLocaleRow(base, "DE", opts)
+}