@mailwoman/corpus 8.5.0 → 9.0.0

package/out/src/tools/sub-venue-lexicon.js

@@ -0,0 +1,817 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Build the sub-venue designator lexicon (#35) — the vocabulary a corpus shard and, eventually, the
+ *   span proposer read to recognize `Terminal 5`, `North Terminal`, `Concourse B`, `ターミナル1` as
+ *   venue-INTERIOR structure.
+ *
+ *   Reads the fetch outputs (`mailwoman corpus fetch wikidata-subvenue`, a JSONL of
+ *   `@mailwoman/osm/sdk`'s `SubVenueSourceRow`s per region, and the Overture slice of `poi.db` via
+ *   `overture-subvenue.ts`) and emits one committed JSON table. The shape follows
+ *   `@mailwoman/poi-taxonomy`'s `taxonomy.json` idiom exactly — typed records plus a FLAT phrase array
+ *   keyed back to a record id, which is what makes a longest-match phrase index cheap to build over it.
+ *   {@link SubVenueSurface} is this table's `SynonymEntry`.
+ *
+ *   ── Determinism ──────────────────────────────────────────────────────────────────────────────────
+ *   {@link buildSubVenueLexicon} is a PURE function of its inputs with a stable sort on every array, so
+ *   a regenerate against the same fetch outputs is byte-identical. No timestamp is emitted for the same
+ *   reason `taxonomy.json` carries none — a clock in the artifact makes every regenerate a diff.
+ *   Vintages live in `sources[]`, taken from the fetch manifests.
+ *
+ *   ── The findings this build is SHAPED BY, all measured ───────────────────────────────────────────
+ *
+ *   **1. A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** Measured on the
+ *   Berlin extract (2,060 matched features, 2026-08-04): a `railway=platform` is named `Stendaler
+ *   Straße`, a `railway=station` is named `Bellevue`, an `amenity=university` is named `Hertie School`.
+ *   Across 250,116 named Great Britain features only 6,003 (2.40%) contain a designator token at all.
+ *   So names are NOT harvested wholesale — {@link extractAttestedPhrases} keeps a name only when it
+ *   CONTAINS a known designator surface, which is what makes `Terminal E (Untere Ebene)` evidence and
+ *   `Otto Lilienthal Flughafen Berlin Tegel` not.
+ *
+ *   **2. The identifier lives in `ref`, not in `name`.** Every one of Berlin's 26 `aeroway=gate`
+ *   features is unnamed and carries only a `ref`: `13`, `6`, `0/1`, `14/15`, `16-18`. That means
+ *   `Gate A12` is a RENDERING (`<designator> <ref>`) rather than a string anyone has written down, and a
+ *   shard that wants to generate the designator+identifier form needs the identifier DISTRIBUTION, not
+ *   a list of phrases. {@link IdentifierShape} is that distribution, and it is why the artifact has a
+ *   section for it at all.
+ *
+ *   **3. A matched phrase belongs to the record the PHRASE names, not to the record the ROW carries.**
+ *   Wave 1 attributed every hit to `row.designatorID`, which is the rule that matched the FEATURE. On
+ *   the GB extract that produced `west → platform`, `hall → platform`, `biggin → platform` — 108 of 133
+ *   OSM-derived surfaces had a `phrase` that named a different record than the one they pointed at,
+ *   because a bus stop tagged `public_transport=platform` is named "Village Hall" or "West Kensington".
+ *   {@link extractAttestedPhrases} now takes a phrase → record INDEX and attributes by phrase; the row's
+ *   own designator is kept as `context`, which is exactly the axis a confound board needs (a `hall` seen
+ *   on a platform is a confound; a `hall` seen on a terminal is evidence).
+ *
+ *   ── What `curated: false` means, and how a surface stops being it ────────────────────────────────
+ *   Wikidata gives a CONCEPT NAME per language, not a designator as addressed. Q849706's Spanish label
+ *   is `terminal aeroportuaria`; the addressed form is `Terminal`. Q240854 (`hall`) is `sala` in
+ *   Italian, which names an ordinary room and would fire on half of Italy. So every machine-derived
+ *   surface lands `curated: false`, and {@link SubVenueLexiconTable} consumers that gate parsing MUST
+ *   filter to `curated: true`.
+ *
+ *   A surface becomes curated ONLY by matching a {@link SubVenuePromotion} in
+ *   `sub-venue-promotions.ts` — a per-designator, per-LOCALE decision carrying the census that backs
+ *   it. Promotion is per-locale because the same token is a designator in one language and a disaster
+ *   in another: `hall` is `Halle 2` at Frankfurt and `Village Hall` at 3,205 British bus stops.
+ *
+ *   ── The seed DUPLICATES `neural/venue-structure.ts`, knowingly ───────────────────────────────────
+ *   `@mailwoman/corpus` does not depend on `@mailwoman/neural` (the dependency runs the other way for
+ *   the training path, and pulling onnxruntime into a corpus build to read three string arrays would be
+ *   absurd), so the shipped vocabulary is re-declared below. That is a drift surface and it is stated
+ *   rather than hidden: `sub-venue-lexicon.test.ts` pins the seed's contents literally, so a change in
+ *   either place fails a test rather than passing silently.
+ */
+import { readFileSync, statSync, writeFileSync } from "node:fs";
+import { basename, join } from "node:path";
+import { parseJSONStrict } from "@mailwoman/core/objects";
+import { TextSpliterator } from "spliterator";
+import { SUBVENUE_PROMOTIONS } from "./sub-venue-promotions.js";
+/**
+ * This table's own data version. Bump when the source vintages or the build semantics change.
+ *
+ * `0.2.0` — wave 2: per-region attestation, the phrase-attribution fix (finding 3 above), derived head nouns, the
+ * Overture source, and the `promotions[]` receipts section.
+ */
+export const SUBVENUE_LEXICON_VERSION = "0.2.0";
+/**
+ * Which side of the containment relation a designator names. Mirrors `@mailwoman/osm/sdk`'s `SubVenueTier`; re-declared
+ * for the same dependency-direction reason as the seed.
+ */
+export const LexiconTier = {
+    SubVenue: "subvenue",
+    Venue: "venue",
+};
+/**
+ * The vocabulary that already ships in `neural/venue-structure.ts`, re-declared. See the module docstring for why this
+ * duplication exists.
+ *
+ * `tier` is added here (the shipped list has no such field): the seven WOF placetypes plus `terminal`/`gate` are all
+ * venue-INTERIOR, except `campus` and `building`, which name a whole venue as often as a part of one. They are marked
+ * `subvenue` anyway, because that is the role the span proposer uses them in — `Building 43, Googleplex` is a unit
+ * inside a venue.
+ */
+export const SHIPPED_DESIGNATOR_SEED = [
+    { id: "arcade", modifierEligible: true, provenance: ["wof:placetype"] },
+    { id: "building", modifierEligible: false, provenance: ["wof:placetype"] },
+    { id: "campus", modifierEligible: true, provenance: ["wof:placetype"] },
+    { id: "concourse", modifierEligible: true, provenance: ["wof:placetype"] },
+    { id: "enclosure", modifierEligible: false, provenance: ["wof:placetype"] },
+    { id: "gate", modifierEligible: false, provenance: ["osm:aeroway=gate"] },
+    { id: "installation", modifierEligible: false, provenance: ["wof:placetype"] },
+    { id: "terminal", modifierEligible: true, provenance: ["osm:aeroway=terminal"] },
+    { id: "wing", modifierEligible: true, provenance: ["wof:placetype"] },
+];
+/**
+ * The shipped positional modifiers, re-declared from `neural/venue-structure.ts`'s `VENUE_STRUCTURE_MODIFIERS`.
+ */
+export const SHIPPED_MODIFIER_SEED = [
+    "central",
+    "east",
+    "front",
+    "inner",
+    "lower",
+    "main",
+    "north",
+    "outer",
+    "rear",
+    "south",
+    "upper",
+    "west",
+];
+/**
+ * Designators the lexicon ADDS beyond what ships, each with the source that attests it.
+ *
+ * `platform`, `station` and `airport` come from the OSM extractor's rule table and are the rail/aviation venue-side
+ * vocabulary the corpus line needs. `hall` and `satellite` come from Wikidata concepts and from
+ * `wof-osm-placetype-map.mdx`'s own "plausible additions" note, which lists `hall` explicitly. `pier` joins them in
+ * wave 2 on 282 Overture attestations in the `pier` category plus 162 in the GB extract — the corpus task names `Pier
+ * C` as a target shape, so the record has to exist before a shard can generate it.
+ *
+ * None is `modifierEligible`: that claim needs a confound board per term AND per locale, and `sub-venue-promotions.ts`
+ * is where those live. A promotion marks a SURFACE usable; it does not widen the modifier grammar.
+ */
+export const PROPOSED_DESIGNATORS = [
+    { id: "airport", tier: LexiconTier.Venue, provenance: ["osm:aeroway=aerodrome"] },
+    { id: "hall", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q240854"] },
+    { id: "pier", tier: LexiconTier.SubVenue, provenance: ["overture:pier"] },
+    { id: "platform", tier: LexiconTier.SubVenue, provenance: ["osm:public_transport=platform", "osm:railway=platform"] },
+    { id: "satellite", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q15990706"] },
+    { id: "station", tier: LexiconTier.Venue, provenance: ["osm:railway=station"] },
+];
+/**
+ * `designatorID` → Wikidata QID, mirroring `fetch/wikidata-subvenue.ts`'s `SUBVENUE_CONCEPTS`. Re-declared here so the
+ * builder stays a pure function over PARSED input rather than reaching into a fetch module for a constant; the test
+ * pins the two against each other.
+ */
+export const CONCEPT_QIDS = {
+    terminal: "Q849706",
+    gate: "Q247739",
+    concourse: "Q862212",
+    campus: "Q209465",
+    building: "Q41176",
+    arcade: "Q186637",
+    hall: "Q240854",
+    satellite: "Q15990706",
+};
+/**
+ * How many real `ref` values each {@link IdentifierShape} keeps.
+ *
+ * Eight, not "all" and not one. The field exists so a shard author can see what a class actually CONTAINS — GB's
+ * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
+ * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
+ * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
+ */
+const IDENTIFIER_EXAMPLES_PER_SHAPE = 8;
+/**
+ * Scripts whose case is meaningful to fold. Everything else is left as written — see {@link SubVenueSurface.phrase}.
+ */
+const CASE_FOLDING_SCRIPT = /^[\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}\d\s\p{P}]+$/u;
+/**
+ * Scripts written without spaces between words, where a token split cannot find a designator and a SUBSTRING test is
+ * the correct operator. Han, Hiragana, Katakana; Hangul is excluded because Korean does space its words.
+ *
+ * The Germanic-compound argument that keeps {@link nameContainsSurface} token-bounded for Latin script does not transfer
+ * here — there is no `-gate`/`-hall` street-name suffix class in Japanese, and `第1ターミナル` is unreachable by any token
+ * split. Measured on the Japan extract: see the harvest counts in `corpus/data/PROVENANCE.md`.
+ */
+const NON_SPACING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
+/**
+ * Normalize a surface for the table: trim, collapse internal whitespace, and lowercase ONLY when the string is entirely
+ * in a bicameral script. `ターミナルビル` and `航站楼` pass through untouched; `Flughafenterminal` folds.
+ */
+export function normalizeSurface(text) {
+    const trimmed = text.trim().replaceAll(/\s+/gu, " ");
+    return CASE_FOLDING_SCRIPT.test(trimmed) ? trimmed.toLowerCase() : trimmed;
+}
+/**
+ * Turn the Wikidata designator-label payload into surfaces.
+ *
+ * A row is dropped when its language tag is empty (an untagged literal, which Wikidata occasionally carries), when the
+ * QID maps to no designator in {@link CONCEPT_QIDS}, or when the normalized phrase is empty. Everything that survives
+ * lands `curated: false` — see the module docstring.
+ */
+export function surfacesFromWikidata(payload, conceptQIDs = CONCEPT_QIDS) {
+    const byQID = new Map(Object.entries(conceptQIDs).map(([id, qid]) => [qid, id]));
+    const envelope = payload;
+    const seen = new Set();
+    const out = [];
+    for (const binding of envelope.results?.bindings ?? []) {
+        const qid = binding.item?.value?.split("/").pop();
+        const recordID = qid ? byQID.get(qid) : undefined;
+        const lang = binding.lang?.value;
+        const raw = binding.label?.value;
+        if (!recordID || !lang || !raw)
+            continue;
+        const phrase = normalizeSurface(raw);
+        if (!phrase)
+            continue;
+        const source = binding.kind?.value === "alt" ? "wikidata:alt" : "wikidata:label";
+        // A concept can carry the same string as both a label and an alias, and across dialect subtags
+        // (`zh`, `zh-cn`, `zh-hans` all say 航站楼). Key the dedupe on the tuple that identifies a row.
+        const key = `${phrase}${recordID}${lang}${source}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push({
+            phrase,
+            recordID,
+            recordKind: "designator",
+            lang,
+            region: "",
+            source,
+            curated: false,
+            observations: 0,
+            context: {},
+        });
+    }
+    return out;
+}
+/**
+ * Classify an OSM `ref` into an {@link IdentifierShape} class.
+ *
+ * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
+ * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
+ * more than one stand: `16-18` and `0/1`.
+ */
+export function classifyIdentifier(ref) {
+    const value = ref.trim();
+    if (/^[0-9]+$/.test(value))
+        return "digit";
+    if (/^[A-Za-z]$/.test(value))
+        return "letter";
+    if (/^[A-Za-z]+[0-9]+$/.test(value))
+        return "letter-digit";
+    if (/^[0-9]+[A-Za-z]+$/.test(value))
+        return "digit-letter";
+    if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value))
+        return "range";
+    return "other";
+}
+/**
+ * Index the surfaces accumulated so far by phrase. First writer wins, so a seed record beats a Wikidata alias that
+ * happens to collide — `terminal` stays the `terminal` designator even though it is also an Italian alias for it.
+ */
+export function buildSurfaceIndex(surfaces) {
+    const index = new Map();
+    for (const surface of surfaces) {
+        if (index.has(surface.phrase))
+            continue;
+        index.set(surface.phrase, { recordID: surface.recordID, recordKind: surface.recordKind });
+    }
+    return index;
+}
+/**
+ * Every known phrase found in `name`, as whole-token runs for spacing scripts and as substrings for non-spacing ones.
+ *
+ * Token-boundary matching for Latin script, not substring: `Nordterminal` is a real German compound in which `terminal`
+ * is a suffix, and a substring test would also fire on `Terminalstraße`. The compound case is a genuine miss and it is
+ * the right miss — admitting suffix matches would fire on every `-hall`/`-gate` compound in Germanic and Nordic street
+ * naming, which is exactly the confound class `Briggate`/`Kirkgate` represents.
+ *
+ * For Han/Kana names that rule finds nothing at all, because the script has no word boundaries: `第1ターミナル` splits into
+ * one token that matches no surface. There the LONGEST known substring is the correct operator, and the compound
+ * objection does not transfer — Japanese has no `-gate` street-name suffix class.
+ */
+export function nameContainsSurfaces(name, index) {
+    const normalized = normalizeSurface(name);
+    const hits = new Set();
+    for (const token of normalized.split(/[\s,()/]+/u)) {
+        const stripped = token.replaceAll(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+        if (stripped && index.has(stripped)) {
+            hits.add(stripped);
+        }
+    }
+    if (NON_SPACING_SCRIPT.test(normalized)) {
+        for (const [phrase] of index) {
+            if (NON_SPACING_SCRIPT.test(phrase) && normalized.includes(phrase)) {
+                hits.add(phrase);
+            }
+        }
+    }
+    return [...hits];
+}
+/**
+ * Harvest attested phrases and identifier shapes out of a source's rows.
+ *
+ * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
+ * is the whole reason this function is safe to run over raw OSM: see the module docstring's finding 1 for the Berlin
+ * measurement that motivated it. The index also decides ATTRIBUTION (finding 3): a hit is a surface of the record the
+ * PHRASE names, and the row's own designator is recorded as `context`.
+ *
+ * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
+ */
+export function extractAttestedPhrases(rows, index, options = {}) {
+    const source = options.source ?? "osm";
+    const region = options.region ?? "";
+    /**
+     * `phrase\0lang\0source` → { count, context }.
+     */
+    const surfaceCounts = new Map();
+    /**
+     * `designatorID\0shape` → { count, examples }.
+     */
+    const shapes = new Map();
+    const note = (phrase, lang, sourceTag, context) => {
+        const key = `${phrase}${lang}${sourceTag}`;
+        const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map() };
+        entry.count++;
+        entry.context.set(context, (entry.context.get(context) ?? 0) + 1);
+        surfaceCounts.set(key, entry);
+    };
+    for (const row of rows) {
+        if (row.name) {
+            for (const hit of nameContainsSurfaces(row.name, index)) {
+                // `und` — the default `name` tag carries no language. Overture's `name` is the same: a
+                // primary name in whatever language the place uses, untagged.
+                note(hit, "und", `${source}:name`, row.designatorID);
+            }
+        }
+        for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
+            for (const hit of nameContainsSurfaces(localized, index)) {
+                note(hit, lang, `${source}:name:${lang}`, row.designatorID);
+            }
+        }
+        if (row.ref) {
+            const shape = classifyIdentifier(row.ref);
+            const key = `${row.designatorID}${shape}`;
+            const entry = shapes.get(key) ?? { count: 0, examples: new Set() };
+            entry.count++;
+            if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
+                entry.examples.add(row.ref.trim());
+            }
+            shapes.set(key, entry);
+        }
+    }
+    const surfaces = [...surfaceCounts].map(([key, entry]) => {
+        const [phrase, lang, sourceTag] = key.split("");
+        const record = index.get(phrase);
+        return {
+            phrase,
+            recordID: record.recordID,
+            recordKind: record.recordKind,
+            lang,
+            region,
+            source: sourceTag,
+            curated: false,
+            observations: entry.count,
+            context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
+        };
+    });
+    const identifierShapes = [...shapes].map(([key, entry]) => {
+        const [designatorID, shape] = key.split("");
+        return {
+            designatorID,
+            region,
+            shape,
+            observations: entry.count,
+            examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
+        };
+    });
+    return { surfaces, identifierShapes };
+}
+/**
+ * Diacritic-flattened ASCII fold, for comparing a Slavic or Turkish inflection against its Latin root.
+ */
+function asciiFold(text) {
+    return text
+        .normalize("NFD")
+        .replaceAll(/\p{Diacritic}/gu, "")
+        .toLowerCase();
+}
+/**
+ * How many leading characters two ASCII-folded forms must share for one to count as the other's inflection.
+ *
+ * Five, or the id's own length when that is shorter (`hall`, `gate`, `wing`, `pier` are four). Measured against the
+ * committed Wikidata pull: at five, `terminal`/`terminál`/`terminale`/`terminali`/`terminála`/`terminalo` are all
+ * accepted for `terminal` while `campo` and `campws` are both rejected for `campus` (they share four). At six the
+ * Spanish `satélite` is lost; at four, Italian `campo` is admitted and it means FIELD.
+ */
+const HEAD_NOUN_PREFIX_FLOOR = 5;
+/**
+ * The shortest substring a non-Latin head-noun candidate may be. Two: `航站` and `터미널` are both real, `楼` alone is
+ * "building" and would fire on every Chinese building name.
+ */
+const NON_LATIN_HEAD_MIN_LENGTH = 2;
+/**
+ * How many head-noun candidates one non-Latin record+language group may contribute. Six — enough to carry `ターミナル`,
+ * `ターミナルビル` and `旅客ターミナル` together, capped because the substring lattice of a nine-character label is large and, ranked
+ * by attesting-surface count, nothing past the sixth has more than the minimum two.
+ */
+const NON_LATIN_HEAD_CANDIDATE_CAP = 6;
+/**
+ * Latin-script test — the scripts an ASCII-folded prefix comparison against a Latin designator id can work on.
+ */
+const LATIN_PHRASE = /^[\p{Script=Latin}\d\s\p{P}]+$/u;
+/**
+ * The scripts the shared-substring derivation is allowed to run on: Han, Hiragana, Katakana, Hangul.
+ *
+ * NARROWER than "not Latin", and the narrowing was earned. Run over every non-Latin phrase in the table, the derivation
+ * produced 90 fragments of Cyrillic, Greek, Arabic, Thai, Burmese and Tamil words — `сгра`, `град`, `κτίρ`,
+ * `ิ่งก่อสร้า` — because those languages have exactly one surface per concept and the only substrings shared inside a
+ * group are pieces of one word. Every one of them was unusable, and none could ever be counted: `poi.db` is four
+ * countries and this wave's extracts are GB, DE, FR, ES and JP, so nothing in reach attests a Thai or Burmese surface.
+ * Deriving a candidate no available source can confirm is not a hypothesis, it is table weight.
+ */
+const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+/**
+ * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
+ * writes on an envelope.
+ *
+ * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
+ * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
+ * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
+ * the head has to be extracted before the curation pass has anything to decide about.
+ *
+ * Two derivations, because the table holds two kinds of writing:
+ *
+ * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
+ *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
+ *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
+ *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
+ *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
+ *   read "Ciudad Universitaria" as sub-venue structure.
+ * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
+ *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
+ *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
+ *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
+ *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
+ *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
+ *   fragment.
+ *
+ * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
+ * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
+ * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
+ * is, and a locale's own data is what confirms or kills it.
+ */
+export function deriveHeadNounSurfaces(surfaces) {
+    const derived = new Map();
+    const seen = new Set(surfaces.map((s) => `${s.phrase}${s.recordID}${s.lang}`));
+    const emit = (phrase, from) => {
+        if (phrase === from.phrase)
+            return;
+        const key = `${phrase}${from.recordID}${from.lang}`;
+        if (seen.has(key) || derived.has(key))
+            return;
+        derived.set(key, {
+            phrase,
+            recordID: from.recordID,
+            recordKind: from.recordKind,
+            lang: from.lang,
+            region: "",
+            source: "derived:head-noun",
+            curated: false,
+            observations: 0,
+            context: {},
+        });
+    };
+    // ── Spacing scripts: prefix-match a token against a single-token surface of the same record ──────
+    // Latin script: a token that is a cognate of the designator's own canonical id.
+    for (const surface of surfaces) {
+        if (!LATIN_PHRASE.test(surface.phrase))
+            continue;
+        const parts = surface.phrase.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
+        if (parts.length < 2)
+            continue;
+        const root = asciiFold(surface.recordID);
+        const floor = Math.min(HEAD_NOUN_PREFIX_FLOOR, root.length);
+        for (const part of parts) {
+            const folded = asciiFold(part);
+            if (folded.length >= floor && commonPrefixLength(folded, root) >= floor) {
+                emit(part, surface);
+            }
+        }
+    }
+    // Non-Latin script: substrings shared by two or more surfaces of the same record + language.
+    const groups = new Map();
+    for (const surface of surfaces) {
+        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
+            continue;
+        // Group `zh`, `zh-cn`, `zh-hant` together: they are writing systems for one vocabulary, and the
+        // simplified/traditional pair is exactly the evidence a shared substring needs.
+        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
+        const pool = groups.get(key) ?? new Set();
+        pool.add(surface.phrase);
+        groups.set(key, pool);
+    }
+    const candidatesByGroup = new Map();
+    for (const [key, pool] of groups) {
+        if (pool.size < 2)
+            continue;
+        candidatesByGroup.set(key, sharedSubstringCandidates(pool));
+    }
+    for (const surface of surfaces) {
+        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
+            continue;
+        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
+        for (const candidate of candidatesByGroup.get(key) ?? []) {
+            if (surface.phrase.includes(candidate)) {
+                emit(candidate, surface);
+            }
+        }
+    }
+    return [...derived.values()];
+}
+/**
+ * Length of the shared leading run of two strings.
+ */
+function commonPrefixLength(a, b) {
+    const limit = Math.min(a.length, b.length);
+    let i = 0;
+    while (i < limit && a[i] === b[i]) {
+        i++;
+    }
+    return i;
+}
+/**
+ * Substrings occurring in at least two DISTINCT members of `pool`, ranked by that count and then by length, capped at
+ * {@link NON_LATIN_HEAD_CANDIDATE_CAP}.
+ *
+ * A candidate never spans whitespace, and in a pool whose members contain whitespace a candidate must be a whole token
+ * of some member. That is what keeps Korean `공항 터미널` from contributing a fragment straddling the space.
+ *
+ * MAXIMAL candidates only: one contained in a longer candidate carried by the SAME number of surfaces is dropped, since
+ * counting can never separate the two. Every one of `ターミナル`'s five ja labels also contains `ターミ`, `ターミナ` and `ミナル`, so
+ * without this the group contributes four indistinguishable candidates and the Japan harvest returns four identical
+ * counts. `航站` survives next to `航站楼` because six surfaces carry it against that one's two.
+ */
+function sharedSubstringCandidates(pool) {
+    const phrases = [...pool];
+    const spaced = phrases.some((phrase) => /\s/u.test(phrase));
+    const tokens = spaced ? new Set(phrases.flatMap((phrase) => phrase.split(/\s+/u).filter(Boolean))) : null;
+    const counts = new Map();
+    for (const phrase of phrases) {
+        const local = new Set();
+        for (let length = NON_LATIN_HEAD_MIN_LENGTH; length <= phrase.length; length++) {
+            for (let start = 0; start + length <= phrase.length; start++) {
+                const candidate = phrase.slice(start, start + length);
+                if (/\s/u.test(candidate))
+                    continue;
+                local.add(candidate);
+            }
+        }
+        for (const candidate of local) {
+            counts.set(candidate, (counts.get(candidate) ?? 0) + 1);
+        }
+    }
+    const kept = [...counts].filter(([candidate, count]) => count >= 2 && (!tokens || tokens.has(candidate)));
+    return kept
+        .filter(([candidate, count]) => kept.every(([other, otherCount]) => other === candidate || otherCount !== count || !other.includes(candidate)))
+        .toSorted((a, b) => b[1] - a[1] || b[0].length - a[0].length || a[0].localeCompare(b[0]))
+        .slice(0, NON_LATIN_HEAD_CANDIDATE_CAP)
+        .map(([candidate]) => candidate);
+}
+/**
+ * Apply the curation decisions to a surface list, IN PLACE on a copy.
+ *
+ * A promotion binds `(designatorID, phrase, locale)`. A surface matches when it names the same record with the same
+ * phrase and its language is the locale's language OR the untagged `und` — the default `name` tag carries no language,
+ * and a German extract's untagged `Halle 2` is German.
+ *
+ * REGION is the subtle half. A surface attested in an extract carries that extract's region and matches only its own
+ * locale. A surface with `region: ""` is region-FREE — a Wikidata label or a derived head noun — and a promotion
+ * reaches it only when no REJECTION exists for the same designator, phrase and language anywhere else. That guard is
+ * not decoration: `pier` is promoted for en-GB and rejected for en-US, and without it the en-GB decision would curate
+ * the region-free English surface and hand `Pier 1 Imports` the promotion en-US was refused. Where no rejection
+ * competes — `terminal` in `es`, `ターミナル` in `ja` — the region-free surface is the whole point, since a language's
+ * designator does not stop at a border.
+ *
+ * Rejections mark nothing themselves. They exist in `promotions[]` as the record of a decision taken, so the next
+ * reader meets en-GB `hall`'s 3,204 bus stops before re-proposing it, not after.
+ */
+export function applyPromotions(surfaces, promotions) {
+    const language = (locale) => locale.split("-")[0];
+    const region = (locale) => locale.split("-")[1] ?? "";
+    const rejectedLanguages = new Set(promotions
+        .filter((promotion) => promotion.decision === "reject")
+        .map((promotion) => `${promotion.designatorID} ${promotion.phrase} ${language(promotion.locale)}`));
+    const promoted = promotions.filter((promotion) => promotion.decision === "promote");
+    return surfaces.map((surface) => {
+        if (surface.curated)
+            return surface;
+        const hit = promoted.some((promotion) => {
+            if (promotion.designatorID !== surface.recordID || promotion.phrase !== surface.phrase)
+                return false;
+            const lang = language(promotion.locale);
+            if (surface.lang !== lang && surface.lang !== "und")
+                return false;
+            if (surface.region === region(promotion.locale))
+                return true;
+            return surface.region === "" && !rejectedLanguages.has(`${promotion.designatorID} ${promotion.phrase} ${lang}`);
+        });
+        return hit ? { ...surface, curated: true } : surface;
+    });
+}
+/**
+ * Build the lexicon table. PURE and deterministic — same inputs, byte-identical output.
+ *
+ * Order of operations is load-bearing in three places:
+ *
+ * 1. Seed surfaces are inserted before anything else, so `terminal` indexes to the `terminal` designator rather than to
+ *    whichever Wikidata alias sorts first.
+ * 2. Head nouns are derived AFTER Wikidata and BEFORE the harvests, because `ターミナル` has to exist as a surface before a
+ *    Japanese extract can be searched for it. That ordering is the entire reason the Japan harvest finds anything — see
+ *    `PROVENANCE.md`.
+ * 3. Promotions are applied LAST, over the union, so a decision can promote a surface whichever source produced it.
+ */
+export function buildSubVenueLexicon(input) {
+    const designators = SHIPPED_DESIGNATOR_SEED.map((seed) => ({
+        id: seed.id,
+        tier: LexiconTier.SubVenue,
+        modifierEligible: seed.modifierEligible,
+        shipped: true,
+        provenance: [...seed.provenance],
+    }));
+    const byID = new Map(designators.map((d) => [d.id, d]));
+    for (const proposed of PROPOSED_DESIGNATORS) {
+        const existing = byID.get(proposed.id);
+        if (existing) {
+            existing.provenance = [...new Set([...existing.provenance, ...proposed.provenance])];
+            continue;
+        }
+        const record = {
+            id: proposed.id,
+            tier: proposed.tier,
+            modifierEligible: false,
+            shipped: false,
+            provenance: [...proposed.provenance],
+        };
+        designators.push(record);
+        byID.set(record.id, record);
+    }
+    // A Wikidata concept id is provenance for the designator it names, whether or not the concept
+    // contributed a usable surface.
+    for (const [id, qid] of Object.entries(CONCEPT_QIDS)) {
+        const record = byID.get(id);
+        if (record) {
+            record.provenance = [...new Set([...record.provenance, `wikidata:${qid}`])];
+        }
+    }
+    const modifiers = SHIPPED_MODIFIER_SEED.map((id) => ({
+        id,
+        shipped: true,
+        provenance: ["codex:directionals"],
+    }));
+    const surfaces = [
+        ...designators.map((d) => ({
+            phrase: d.id,
+            recordID: d.id,
+            recordKind: "designator",
+            lang: "en",
+            region: "",
+            source: "seed",
+            // The English designator IS the shipped vocabulary — curated by construction.
+            curated: d.shipped,
+            observations: 0,
+            context: {},
+        })),
+        ...modifiers.map((m) => ({
+            phrase: m.id,
+            recordID: m.id,
+            recordKind: "modifier",
+            lang: "en",
+            region: "",
+            source: "seed",
+            curated: true,
+            observations: 0,
+            context: {},
+        })),
+    ];
+    if (input.wikidata) {
+        surfaces.push(...surfacesFromWikidata(input.wikidata));
+    }
+    surfaces.push(...deriveHeadNounSurfaces(surfaces));
+    const identifierShapes = [];
+    for (const harvest of input.harvests) {
+        const attested = extractAttestedPhrases(harvest.rows, buildSurfaceIndex(surfaces), {
+            source: harvest.source,
+            region: harvest.region,
+        });
+        surfaces.push(...attested.surfaces);
+        identifierShapes.push(...attested.identifierShapes);
+    }
+    const promotions = [...(input.promotions ?? SUBVENUE_PROMOTIONS)];
+    const curated = applyPromotions(surfaces, promotions);
+    // Deterministic order everywhere. `localeCompare` matches the tie-break discipline
+    // `generate-taxonomy.ts` and `build-brands.ts` already use.
+    designators.sort((a, b) => a.id.localeCompare(b.id));
+    modifiers.sort((a, b) => a.id.localeCompare(b.id));
+    curated.sort((a, b) => a.phrase.localeCompare(b.phrase) ||
+        a.recordID.localeCompare(b.recordID) ||
+        a.lang.localeCompare(b.lang) ||
+        a.region.localeCompare(b.region) ||
+        a.source.localeCompare(b.source));
+    return {
+        version: SUBVENUE_LEXICON_VERSION,
+        sources: input.sources.toSorted((a, b) => a.id.localeCompare(b.id)),
+        designators,
+        modifiers,
+        surfaces: curated,
+        identifierShapes: identifierShapes.toSorted((a, b) => a.designatorID.localeCompare(b.designatorID) ||
+            a.region.localeCompare(b.region) ||
+            a.shape.localeCompare(b.shape)),
+        promotions: promotions.toSorted((a, b) => a.designatorID.localeCompare(b.designatorID) ||
+            a.locale.localeCompare(b.locale) ||
+            a.phrase.localeCompare(b.phrase)),
+    };
+}
+/**
+ * Serialize the table the way the committed artifact stores it: pretty-printed, trailing newline. Run `oxfmt` over the
+ * result before committing — repo law is that committed JSON is oxfmt-clean, which `JSON.stringify` cannot reproduce.
+ */
+export function serializeSubVenueLexicon(table) {
+    return JSON.stringify(table, null, 2) + "\n";
+}
+/**
+ * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
+ * extract is a build output, and one malformed line should not cost the whole lexicon.
+ */
+export function readSubVenueJSONL(path) {
+    const out = [];
+    // `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
+    // (52 MB for Great Britain), and materializing every segment before reading the first is exactly
+    // what the repo lint rule exists to prevent.
+    for (const line of TextSpliterator.from(readFileSync(path, "utf8"))) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            out.push(parseJSONStrict(trimmed));
+        }
+        catch {
+            continue;
+        }
+    }
+    return out;
+}
+/**
+ * Read the fetch outputs, build the table, and write it.
+ *
+ * The IO half only — every decision lives in {@link buildSubVenueLexicon}, which is pure. Run `oxfmt` over `outPath`
+ * afterwards; repo law is that committed JSON is oxfmt-clean.
+ */
+export function generateSubVenueLexicon(options) {
+    const sources = [];
+    const harvests = [];
+    let wikidata = null;
+    if (options.wikidataDir) {
+        wikidata = parseJSONStrict(readFileSync(join(options.wikidataDir, "designator-labels.json"), "utf8"));
+        const manifest = parseJSONStrict(readFileSync(join(options.wikidataDir, "MANIFEST.json"), "utf8"));
+        const labelFile = manifest.files?.find((f) => f.filename === "designator-labels.json");
+        sources.push({
+            id: "wikidata",
+            origin: manifest.endpoint ?? "https://query.wikidata.org/sparql",
+            license: manifest.license ?? "CC0",
+            // The DATE only. A full ISO timestamp would make every re-fetch a diff in the committed artifact for
+            // no information a reader of a vocabulary table can act on.
+            retrieved: (manifest.downloaded_at ?? "").slice(0, 10),
+            rows: labelFile?.rows ?? 0,
+        });
+    }
+    for (const extract of options.extracts ?? []) {
+        const rows = readSubVenueJSONL(extract.path);
+        harvests.push({ rows, source: "osm", region: extract.region });
+        sources.push({
+            id: `osm:${extract.region.toLowerCase()}`,
+            // The extract's NAME, never its path. `AGENTS.md` forbids re-hardcoding the lab data root
+            // anywhere, and a committed artifact carrying `/mnt/playpen/...` would do exactly that while
+            // telling a reader on another machine nothing. `great-britain` identifies the Geofabrik region,
+            // which is the fact that matters.
+            origin: `OpenStreetMap via Geofabrik (${basename(extract.path, ".jsonl")})`,
+            license: "ODbL (OpenStreetMap)",
+            // The extract's mtime — when the rows were produced. `corpus/AGENTS.md`'s standing warning that
+            // a file's mtime is not its DATA's vintage applies to a downloaded archive; this file is a build
+            // output of ours, so its mtime is exactly the right number.
+            retrieved: statSync(extract.path).mtime.toISOString().slice(0, 10),
+            rows: rows.length,
+        });
+    }
+    if (options.overtureRows?.length) {
+        // Overture rows carry their own country, so they are harvested per REGION rather than in one pass —
+        // a `region` on the surface is the axis promotion is decided on and a mixed-country bucket would
+        // make it meaningless.
+        const byCountry = new Map();
+        for (const row of options.overtureRows) {
+            const bucket = byCountry.get(row.country) ?? [];
+            bucket.push(row);
+            byCountry.set(row.country, bucket);
+        }
+        for (const [country, rows] of [...byCountry].toSorted((a, b) => a[0].localeCompare(b[0]))) {
+            harvests.push({ rows, source: "overture", region: country });
+        }
+        sources.push({
+            id: "overture",
+            origin: "Overture Maps Foundation places, via the poi.db spatial layer",
+            license: "CDLA-Permissive-2.0",
+            retrieved: options.overtureVintage ?? "",
+            rows: options.overtureRows.length,
+        });
+    }
+    const table = buildSubVenueLexicon({ wikidata, harvests, sources });
+    writeFileSync(options.outPath, serializeSubVenueLexicon(table));
+    return table;
+}
+//# sourceMappingURL=sub-venue-lexicon.js.map