@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/tools/postcode-triples.ts

@@ -41,6 +41,8 @@
  *   recipe's header for the measurement.
  */
 
+import { officialLanguagesAlpha3, regionLanguagesAlpha3 } from "@mailwoman/codex/country"
+import { foldName } from "@mailwoman/codex/normalize"
 import { dataRootPath } from "@mailwoman/core/data-root"
 import { pathExists } from "@mailwoman/core/fs/readers"
 import type { WOFDatabase } from "@mailwoman/resolver-wof-sqlite/schema"
@@ -48,7 +50,7 @@ import { DatabaseClient } from "@mailwoman/sqlite/client"
 import { join } from "path-ts"
 import { TSVSpliterator } from "spliterator"
 
-import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames-postal/adapter"
+import { GEONAMES_POSTAL_COLUMNS } from "#adapters/geonames/postal/adapter"
 import type { PostcodePlacement } from "#recipes/scaffold"
 import { escapeSQLString } from "#utils/parquet"
 
@@ -172,6 +174,61 @@ export function applyCountryBudget(
 	return kept
 }
 
+/**
+ * A place's preferred names in the languages its addresses are written in, read from the gazetteer's `names` table.
+ */
+interface PreferredNames {
+	/**
+	 * Preferred names in the country's official languages, in the table's order.
+	 */
+	official: readonly string[]
+	/**
+	 * Preferred names in the region's co-official languages.
+	 */
+	coOfficial: readonly string[]
+}
+
+/**
+ * The surfaces a REGION is written as: its preferred names in the official language(s), then in the region's
+ * co-official languages, then the gazetteer's own `spr.name` (the English exonym). Deduplicated, order kept.
+ *
+ * `spr.name` is the English-preferred, diacritic-stripped label, and reading it alone is the defect #1673 measured:
+ * 53,078 ES rows teaching `Balearic Islands` (2,872 `Andalusia`, 5,680 `Castile and Leon`) against 4 rows of `Illes
+ * Balears`, and every province with its accent gone (`Cordoba`, `Leon`). A Spanish user writes `Islas Baleares` or
+ * `Illes Balears`; the exonym stays as ONE surface among them because a user may write it too.
+ */
+export function regionWrittenForms(sprName: string, names: PreferredNames): string[] {
+	const out: string[] = []
+
+	for (const raw of [...names.official, ...names.coOfficial, sprName]) {
+		const name = raw.replace(PROVINCE_GENERIC, "")
+
+		if (name && !out.includes(name)) {
+			out.push(name)
+		}
+	}
+
+	return out
+}
+
+/**
+ * The provincial generic Who's On First keeps in a province's Catalan and Asturian preferred names (`Província de
+ * Barcelona`, `Província d'A Coruña`). An envelope carries the name alone, so the generic is dropped and the surface
+ * dedupes against the Castilian one.
+ */
+const PROVINCE_GENERIC = /^prov[ií]ncia (?:de |d')/iu
+
+/**
+ * The surface a LOCALITY is written as: the official-language preferred name that is `spr.name` with its diacritics
+ * restored (`Cordoba` → `Córdoba`), else the first official-language preferred name, else `spr.name`. One surface, not
+ * a fan-out: the region carries the multiplicity and the locality would multiply it.
+ */
+export function localityWrittenForm(sprName: string, names: PreferredNames): string {
+	const folded = foldName(sprName)
+
+	return names.official.find((name) => foldName(name) === folded) ?? names.official[0] ?? sprName
+}
+
 /**
  * Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
  * place's ancestry to a region.
@@ -179,6 +236,12 @@ export function applyCountryBudget(
  * A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
  * a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
  * actually reachable.
+ *
+ * One triple per REGION SURFACE (see {@link regionWrittenForms}): a Balearic postcode yields `Islas Baleares`, `Illes
+ * Balears` and `Balearic Islands` rows, a Zamora one `Zamora` alone. The languages come from the codex — the country's
+ * official languages plus the province's co-official ones — and never from the names table's own language list, whose
+ * "preferred" name in a language not spoken in the province is often the parent community's (`Zamora` → `Castella i
+ * Lleó`).
  */
 export async function readTriplesFromParentJoin(
 	countries: readonly string[],
@@ -194,7 +257,8 @@ export async function readTriplesFromParentJoin(
 	db.exec(`ATTACH DATABASE '${escapeSQLString(postcodeDB)}' AS pc`)
 
 	const statement = db.prepare(`
-		SELECT p.name AS postcode, a.name AS locality, r.name AS region, c.name AS country, p.country AS cc
+		SELECT p.name AS postcode, a.id AS locality_id, a.name AS locality, r.id AS region_id, r.name AS region,
+		       c.name AS country, p.country AS cc
 		FROM pc.spr p
 		JOIN spr a ON a.id = p.parent_id AND a.placetype IN ('locality', 'localadmin')
 		JOIN ancestors anc ON anc.id = a.id AND anc.ancestor_placetype = 'region'
@@ -205,6 +269,37 @@ export async function readTriplesFromParentJoin(
 		ORDER BY p.id
 	`)
 
+	const preferredStatement = db.prepare(
+		`SELECT language, name FROM names WHERE id = ? AND privateuse = 'preferred' AND language IS NOT NULL ORDER BY language, name`
+	)
+
+	const preferredByID = new Map<number, Map<string, string[]>>()
+
+	/**
+	 * Preferred names of one place, grouped by language, read once per id.
+	 */
+	const preferredNames = (id: number): Map<string, string[]> => {
+		let byLanguage = preferredByID.get(id)
+
+		if (!byLanguage) {
+			byLanguage = new Map()
+
+			for (const row of preferredStatement.all(id) as Array<{ language: string; name: string }>) {
+				const list = byLanguage.get(row.language) ?? []
+
+				list.push(row.name)
+				byLanguage.set(row.language, list)
+			}
+
+			preferredByID.set(id, byLanguage)
+		}
+
+		return byLanguage
+	}
+
+	const namesIn = (byLanguage: Map<string, string[]>, languages: readonly string[]): string[] =>
+		languages.flatMap((language) => byLanguage.get(language) ?? [])
+
 	const out: PostcodeTriple[] = []
 
 	for (const cc of countries) {
@@ -212,18 +307,47 @@ export async function readTriplesFromParentJoin(
 
 		if (!convention) continue
 
-		for (const row of statement.all(cc) as Array<Record<string, string>>) {
-			if (!row["postcode"] || !row["locality"] || !row["region"]) continue
-
-			out.push({
-				postcode: row["postcode"],
-				locality: row["locality"],
-				region: row["region"],
-				country: row["country"] ?? "",
-				cc,
-				locale: convention.locale,
-				postcodePlacement: convention.placement,
-			})
+		const officialLanguages = officialLanguagesAlpha3(cc)
+
+		for (const row of statement.all(cc) as Array<{
+			postcode: string | null
+			locality_id: number
+			locality: string | null
+			region_id: number
+			region: string | null
+			country: string | null
+		}>) {
+			if (!row.postcode || !row.locality || !row.region) continue
+
+			const regionOfficial = namesIn(preferredNames(row.region_id), officialLanguages)
+			// The co-official table is keyed by the region's name in the first official language; a region the names
+			// table has no such name for is looked up by its `spr.name`, which for a monolingual country is the same string.
+			const regionLanguages = regionLanguagesAlpha3(cc, regionOfficial[0] ?? row.region)
+			const regionCoOfficialLanguages = regionLanguages.filter((language) => !officialLanguages.includes(language))
+
+			const regionNames: PreferredNames = {
+				official: regionOfficial,
+				coOfficial: namesIn(preferredNames(row.region_id), regionCoOfficialLanguages),
+			}
+
+			const localityNames: PreferredNames = {
+				official: namesIn(preferredNames(row.locality_id), officialLanguages),
+				coOfficial: [],
+			}
+
+			const locality = localityWrittenForm(row.locality, localityNames)
+
+			for (const region of regionWrittenForms(row.region, regionNames)) {
+				out.push({
+					postcode: row.postcode,
+					locality,
+					region,
+					country: row.country ?? "",
+					cc,
+					locale: convention.locale,
+					postcodePlacement: convention.placement,
+				})
+			}
 		}
 	}
 

package/lib/tools/sub/venue/harvest.ts

@@ -0,0 +1,234 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
+ *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
+ *   measured shape of each designator's identifier half.
+ *
+ *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
+ *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
+ *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
+ *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ *
+ *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
+ *   over raw OSM at all:
+ *
+ *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
+ *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
+ *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
+ *   designator token at all. So names are not harvested wholesale — a name contributes only when it
+ *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
+ *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
+ *
+ *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
+ *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
+ *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
+ *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
+ *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
+ *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
+ *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
+ */
+
+import { readLocalTextFile } from "@mailwoman/core/fs/readers"
+import { parseJSONStrict } from "@mailwoman/core/json"
+import { TextSpliterator } from "spliterator"
+
+import { nameContainsSurfaces, type SurfaceIndex } from "#tools/sub/venue/surfaces"
+import type { IdentifierShape, SubVenueSurface } from "#tools/sub/venue/table"
+
+/**
+ * One row of a harvestable source, as read back off JSONL or out of a layer database.
+ *
+ * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
+ * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
+ * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
+ * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ */
+export interface SubVenueHarvestRow {
+	/**
+	 * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
+	 * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
+	 */
+	designatorID: string
+	/**
+	 * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
+	 * read it; the sub-venue slice recipe fills its venue slot from it.
+	 */
+	tier?: string
+	name?: string | null
+	ref?: string | null
+	localizedNames?: Record<string, string>
+}
+
+/**
+ * Classify an OSM `ref` into an {@link IdentifierShape} class.
+ *
+ * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
+ * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
+ * more than one stand: `16-18` and `0/1`.
+ */
+export function classifyIdentifier(ref: string): string {
+	const value = ref.trim()
+
+	if (/^[0-9]+$/.test(value)) return "digit"
+
+	if (/^[A-Za-z]$/.test(value)) return "letter"
+
+	if (/^[A-Za-z]+[0-9]+$/.test(value)) return "letter-digit"
+
+	if (/^[0-9]+[A-Za-z]+$/.test(value)) return "digit-letter"
+
+	if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value)) return "range"
+
+	return "other"
+}
+
+/**
+ * How many real `ref` values each {@link IdentifierShape} keeps.
+ *
+ * Eight, not "all" and not one. The field exists so a slice author can see what a class actually CONTAINS — GB's
+ * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
+ * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
+ * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
+ */
+const IDENTIFIER_EXAMPLES_PER_SHAPE = 8
+
+/**
+ * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
+ *
+ * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
+ */
+export interface HarvestOptions {
+	/**
+	 * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
+	 */
+	source?: string
+	/**
+	 * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
+	 */
+	region?: string
+}
+
+/**
+ * Harvest attested phrases and identifier shapes out of a source's rows.
+ *
+ * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
+ * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
+ * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
+ * own designator is recorded as `context`.
+ *
+ * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
+ */
+export function extractAttestedPhrases(
+	rows: Iterable<SubVenueHarvestRow>,
+	index: SurfaceIndex,
+	options: HarvestOptions = {}
+): { surfaces: SubVenueSurface[]; identifierShapes: IdentifierShape[] } {
+	const source = options.source ?? "osm"
+	const region = options.region ?? ""
+	/**
+	 * `phrase\0lang\0source` → { count, context }.
+	 */
+	const surfaceCounts = new Map<string, { count: number; context: Map<string, number> }>()
+	/**
+	 * `designatorID\0shape` → { count, examples }.
+	 */
+	const shapes = new Map<string, { count: number; examples: Set<string> }>()
+
+	const note = (phrase: string, lang: string, sourceTag: string, context: string): void => {
+		const key = `${phrase}\0${lang}\0${sourceTag}`
+		const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map<string, number>() }
+
+		entry.count++
+		entry.context.set(context, (entry.context.get(context) ?? 0) + 1)
+		surfaceCounts.set(key, entry)
+	}
+
+	for (const row of rows) {
+		if (row.name) {
+			for (const hit of nameContainsSurfaces(row.name, index)) {
+				// `und` — the default `name` tag carries no language. Overture's `name` is the same: a
+				// primary name in whatever language the place uses, untagged.
+				note(hit, "und", `${source}:name`, row.designatorID)
+			}
+		}
+
+		for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
+			for (const hit of nameContainsSurfaces(localized, index)) {
+				note(hit, lang, `${source}:name:${lang}`, row.designatorID)
+			}
+		}
+
+		if (row.ref) {
+			const shape = classifyIdentifier(row.ref)
+			const key = `${row.designatorID}\0${shape}`
+			const entry = shapes.get(key) ?? { count: 0, examples: new Set<string>() }
+
+			entry.count++
+
+			if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
+				entry.examples.add(row.ref.trim())
+			}
+
+			shapes.set(key, entry)
+		}
+	}
+
+	const surfaces: SubVenueSurface[] = [...surfaceCounts].map(([key, entry]) => {
+		const [phrase, lang, sourceTag] = key.split("\0") as [string, string, string]
+		const record = index.get(phrase)!
+
+		return {
+			phrase,
+			recordID: record.recordID,
+			recordKind: record.recordKind,
+			lang,
+			region,
+			source: sourceTag,
+			curated: false,
+			observations: entry.count,
+			context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
+		}
+	})
+
+	const identifierShapes: IdentifierShape[] = [...shapes].map(([key, entry]) => {
+		const [designatorID, shape] = key.split("\0") as [string, string]
+
+		return {
+			designatorID,
+			region,
+			shape,
+			observations: entry.count,
+			examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
+		}
+	})
+
+	return { surfaces, identifierShapes }
+}
+
+/**
+ * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
+ * extract is a build output, and one malformed line should not cost the whole lexicon.
+ */
+export async function readSubVenueJSONL(path: string): Promise<SubVenueHarvestRow[]> {
+	const out: SubVenueHarvestRow[] = []
+
+	// `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
+	// (52 MB for Great Britain), and materializing every segment before reading the first is exactly
+	// what the repo lint rule exists to prevent.
+	for (const line of TextSpliterator.from(await readLocalTextFile(path))) {
+		const trimmed = line.trim()
+
+		if (!trimmed) continue
+
+		try {
+			out.push(parseJSONStrict<SubVenueHarvestRow>(trimmed))
+		} catch {
+			continue
+		}
+	}
+
+	return out
+}

package/lib/tools/sub/venue/head-nouns.ts

@@ -0,0 +1,245 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Head-noun derivation for the sub-venue lexicon — proposing the ADDRESSED form of a designator
+ *   from the encyclopaedic label a vocabulary source carries.
+ *
+ *   Wikidata's label for a concept is the encyclopaedic name (`terminal aeroportuaria`, `letištní
+ *   terminál`, `havalimanı terminali`), while the addressed form is the bare head (`Terminal`,
+ *   `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so the head has to be
+ *   extracted before the curation pass has anything to decide about — that gap is why the first wave of
+ *   this table shipped 1,014 uncurated surfaces.
+ *
+ *   Everything derived lands `curated: false`. A derivation is a HYPOTHESIS about what the addressed
+ *   form is; a locale's own data is what confirms or kills it.
+ */
+
+import { isPresent } from "@mailwoman/core/objects"
+
+import type { SubVenueSurface } from "#tools/sub/venue/table"
+
+/**
+ * Diacritic-flattened ASCII fold, for comparing a Slavic or Turkish inflection against its Latin root.
+ *
+ * Deliberately `\p{Diacritic}` rather than `@mailwoman/normalize/fold`'s `stripCombiningMarks` (`\p{M}`): the two
+ * classes diverge outside plain diacritics, and {@link HEAD_NOUN_PREFIX_FLOOR} was calibrated against this exact fold.
+ */
+function asciiFold(text: string): string {
+	return text
+		.normalize("NFD")
+		.replaceAll(/\p{Diacritic}/gu, "")
+		.toLowerCase()
+}
+
+/**
+ * How many leading characters two ASCII-folded forms must share for one to count as the other's inflection.
+ *
+ * Five, or the id's own length when that is shorter (`hall`, `gate`, `wing`, `pier` are four). Measured against the
+ * committed Wikidata pull: at five, `terminal`/`terminál`/`terminale`/`terminali`/`terminála`/`terminalo` are all
+ * accepted for `terminal` while `campo` and `campws` are both rejected for `campus` (they share four). At six the
+ * Spanish `satélite` is lost; at four, Italian `campo` is admitted and it means FIELD.
+ */
+const HEAD_NOUN_PREFIX_FLOOR = 5
+
+/**
+ * The shortest substring a non-Latin head-noun candidate may be. Two: `航站` and `터미널` are both real, `楼` alone is
+ * "building" and would fire on every Chinese building name.
+ */
+const NON_LATIN_HEAD_MIN_LENGTH = 2
+
+/**
+ * How many head-noun candidates one non-Latin record+language group may contribute. Six — enough to carry `ターミナル`,
+ * `ターミナルビル` and `旅客ターミナル` together, capped because the substring lattice of a nine-character label is large and, ranked
+ * by attesting-surface count, nothing past the sixth has more than the minimum two.
+ */
+const NON_LATIN_HEAD_CANDIDATE_CAP = 6
+
+/**
+ * Latin-script test — the scripts an ASCII-folded prefix comparison against a Latin designator id can work on.
+ */
+const LATIN_PHRASE = /^[\p{Script=Latin}\d\s\p{P}]+$/u
+
+/**
+ * The scripts the shared-substring derivation is allowed to run on: Han, Hiragana, Katakana, Hangul.
+ *
+ * NARROWER than "not Latin", and the narrowing was earned. Run over every non-Latin phrase in the table, the derivation
+ * produced 90 fragments of Cyrillic, Greek, Arabic, Thai, Burmese and Tamil words — `сгра`, `град`, `κτίρ`,
+ * `ิ่งก่อสร้า` — because those languages have exactly one surface per concept and the only substrings shared inside a
+ * group are pieces of one word. Every one of them was unusable, and none could ever be counted: `poi.db` is four
+ * countries and this wave's extracts are GB, DE, FR, ES and JP, so nothing in reach attests a Thai or Burmese surface.
+ * Deriving a candidate no available source can confirm is not a hypothesis, it is table weight.
+ */
+const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u
+
+/**
+ * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
+ * writes on an envelope.
+ *
+ * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
+ * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
+ * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
+ * the head has to be extracted before the curation pass has anything to decide about.
+ *
+ * Two derivations, because the table holds two kinds of writing:
+ *
+ * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
+ *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
+ *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
+ *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
+ *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
+ *   read "Ciudad Universitaria" as sub-venue structure.
+ * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
+ *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
+ *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
+ *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
+ *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
+ *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
+ *   fragment.
+ *
+ * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
+ * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
+ * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
+ * is, and a locale's own data is what confirms or kills it.
+ */
+export function deriveHeadNounSurfaces(surfaces: readonly SubVenueSurface[]): SubVenueSurface[] {
+	const derived = new Map<string, SubVenueSurface>()
+	const seen = new Set(surfaces.map((s) => `${s.phrase}\0${s.recordID}\0${s.lang}`))
+
+	const emit = (phrase: string, from: SubVenueSurface): void => {
+		if (phrase === from.phrase) return
+
+		const key = `${phrase}\0${from.recordID}\0${from.lang}`
+
+		if (seen.has(key) || derived.has(key)) return
+
+		derived.set(key, {
+			phrase,
+			recordID: from.recordID,
+			recordKind: from.recordKind,
+			lang: from.lang,
+			region: "",
+			source: "derived:head-noun",
+			curated: false,
+			observations: 0,
+			context: {},
+		})
+	}
+
+	// ── Spacing scripts: prefix-match a token against a single-token surface of the same record ──────
+	// Latin script: a token that is a cognate of the designator's own canonical id.
+	for (const surface of surfaces) {
+		if (!LATIN_PHRASE.test(surface.phrase)) continue
+
+		const parts = surface.phrase.split(/[^\p{L}\p{N}]+/u).filter(isPresent)
+
+		if (parts.length < 2) continue
+
+		const root = asciiFold(surface.recordID)
+		const floor = Math.min(HEAD_NOUN_PREFIX_FLOOR, root.length)
+
+		for (const part of parts) {
+			const folded = asciiFold(part)
+
+			if (folded.length >= floor && commonPrefixLength(folded, root) >= floor) {
+				emit(part, surface)
+			}
+		}
+	}
+
+	// Non-Latin script: substrings shared by two or more surfaces of the same record + language.
+	const groups = new Map<string, Set<string>>()
+
+	for (const surface of surfaces) {
+		if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase)) continue
+
+		// Group `zh`, `zh-cn`, `zh-hant` together: they are writing systems for one vocabulary, and the
+		// simplified/traditional pair is exactly the evidence a shared substring needs.
+		const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]!}`
+		const pool = groups.get(key) ?? new Set<string>()
+		pool.add(surface.phrase)
+		groups.set(key, pool)
+	}
+
+	const candidatesByGroup = new Map<string, string[]>()
+
+	for (const [key, pool] of groups) {
+		if (pool.size < 2) continue
+		candidatesByGroup.set(key, sharedSubstringCandidates(pool))
+	}
+
+	for (const surface of surfaces) {
+		if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase)) continue
+
+		const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]!}`
+
+		for (const candidate of candidatesByGroup.get(key) ?? []) {
+			if (surface.phrase.includes(candidate)) {
+				emit(candidate, surface)
+			}
+		}
+	}
+
+	return [...derived.values()]
+}
+
+/**
+ * Length of the shared leading run of two strings.
+ */
+function commonPrefixLength(a: string, b: string): number {
+	const limit = Math.min(a.length, b.length)
+	let i = 0
+
+	while (i < limit && a[i] === b[i]) {
+		i++
+	}
+
+	return i
+}
+
+/**
+ * Substrings occurring in at least two DISTINCT members of `pool`, ranked by that count and then by length, capped at
+ * {@link NON_LATIN_HEAD_CANDIDATE_CAP}.
+ *
+ * A candidate never spans whitespace, and in a pool whose members contain whitespace a candidate must be a whole token
+ * of some member. That is what keeps Korean `공항 터미널` from contributing a fragment straddling the space.
+ *
+ * MAXIMAL candidates only: one contained in a longer candidate carried by the SAME number of surfaces is dropped, since
+ * counting can never separate the two. Every one of `ターミナル`'s five ja labels also contains `ターミ`, `ターミナ` and `ミナル`, so
+ * without this the group contributes four indistinguishable candidates and the Japan harvest returns four identical
+ * counts. `航站` survives next to `航站楼` because six surfaces carry it against that one's two.
+ */
+function sharedSubstringCandidates(pool: ReadonlySet<string>): string[] {
+	const phrases = [...pool]
+	const spaced = phrases.some((phrase) => /\s/u.test(phrase))
+	const tokens = spaced ? new Set(phrases.flatMap((phrase) => phrase.split(/\s+/u).filter(isPresent))) : null
+	const counts = new Map<string, number>()
+
+	for (const phrase of phrases) {
+		const local = new Set<string>()
+
+		for (let length = NON_LATIN_HEAD_MIN_LENGTH; length <= phrase.length; length++) {
+			for (let start = 0; start + length <= phrase.length; start++) {
+				const candidate = phrase.slice(start, start + length)
+
+				if (/\s/u.test(candidate)) continue
+				local.add(candidate)
+			}
+		}
+
+		for (const candidate of local) {
+			counts.set(candidate, (counts.get(candidate) ?? 0) + 1)
+		}
+	}
+
+	const kept = [...counts].filter(([candidate, count]) => count >= 2 && (!tokens || tokens.has(candidate)))
+
+	return kept
+		.filter(([candidate, count]) =>
+			kept.every(([other, otherCount]) => other === candidate || otherCount !== count || !other.includes(candidate))
+		)
+		.toSorted((a, b) => b[1] - a[1] || b[0].length - a[0].length || a[0].localeCompare(b[0]))
+		.slice(0, NON_LATIN_HEAD_CANDIDATE_CAP)
+		.map(([candidate]) => candidate)
+}