@mailwoman/corpus 9.0.0 → 9.2.0

package/out/src/tools/sub-venue-lexicon.js

@@ -3,16 +3,14 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Build the sub-venue designator lexicon (#35) — the vocabulary a corpus shard and, eventually, the
- *   span proposer read to recognize `Terminal 5`, `North Terminal`, `Concourse B`, `ターミナル1` as
- *   venue-INTERIOR structure.
+ *   @file Build the sub-venue designator lexicon (#35) — the vocabulary a corpus shard and, eventually,
+ *   the span proposer read to recognize `Terminal 5`, `North Terminal`, `Concourse B`, `ターミナル1` as
+ *   venue-INTERIOR structure. This is the assembly: the record schema lives in `sub-venue/table.ts`, the
+ *   per-stage machinery in its siblings, and the curation decisions in `sub-venue-promotions.ts`.
  *
  *   Reads the fetch outputs (`mailwoman corpus fetch wikidata-subvenue`, a JSONL of
  *   `@mailwoman/osm/sdk`'s `SubVenueSourceRow`s per region, and the Overture slice of `poi.db` via
- *   `overture-subvenue.ts`) and emits one committed JSON table. The shape follows
- *   `@mailwoman/poi-taxonomy`'s `taxonomy.json` idiom exactly — typed records plus a FLAT phrase array
- *   keyed back to a record id, which is what makes a longest-match phrase index cheap to build over it.
- *   {@link SubVenueSurface} is this table's `SynonymEntry`.
+ *   `overture-subvenue.ts`) and emits one committed JSON table.
  *
  *   ── Determinism ──────────────────────────────────────────────────────────────────────────────────
  *   {@link buildSubVenueLexicon} is a PURE function of its inputs with a stable sort on every array, so
@@ -20,550 +18,39 @@
  *   reason `taxonomy.json` carries none — a clock in the artifact makes every regenerate a diff.
  *   Vintages live in `sources[]`, taken from the fetch manifests.
  *
- *   ── The findings this build is SHAPED BY, all measured ───────────────────────────────────────────
+ *   ── Where the stages live ────────────────────────────────────────────────────────────────────────
+ *   Each stage carries the measurements that shaped it; the order they run in is
+ *   {@link buildSubVenueLexicon}'s own docstring, and it is load-bearing.
  *
- *   **1. A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** Measured on the
- *   Berlin extract (2,060 matched features, 2026-08-04): a `railway=platform` is named `Stendaler
- *   Straße`, a `railway=station` is named `Bellevue`, an `amenity=university` is named `Hertie School`.
- *   Across 250,116 named Great Britain features only 6,003 (2.40%) contain a designator token at all.
- *   So names are NOT harvested wholesale — {@link extractAttestedPhrases} keeps a name only when it
- *   CONTAINS a known designator surface, which is what makes `Terminal E (Untere Ebene)` evidence and
- *   `Otto Lilienthal Flughafen Berlin Tegel` not.
- *
- *   **2. The identifier lives in `ref`, not in `name`.** Every one of Berlin's 26 `aeroway=gate`
- *   features is unnamed and carries only a `ref`: `13`, `6`, `0/1`, `14/15`, `16-18`. That means
- *   `Gate A12` is a RENDERING (`<designator> <ref>`) rather than a string anyone has written down, and a
- *   shard that wants to generate the designator+identifier form needs the identifier DISTRIBUTION, not
- *   a list of phrases. {@link IdentifierShape} is that distribution, and it is why the artifact has a
- *   section for it at all.
- *
- *   **3. A matched phrase belongs to the record the PHRASE names, not to the record the ROW carries.**
- *   Wave 1 attributed every hit to `row.designatorID`, which is the rule that matched the FEATURE. On
- *   the GB extract that produced `west → platform`, `hall → platform`, `biggin → platform` — 108 of 133
- *   OSM-derived surfaces had a `phrase` that named a different record than the one they pointed at,
- *   because a bus stop tagged `public_transport=platform` is named "Village Hall" or "West Kensington".
- *   {@link extractAttestedPhrases} now takes a phrase → record INDEX and attributes by phrase; the row's
- *   own designator is kept as `context`, which is exactly the axis a confound board needs (a `hall` seen
- *   on a platform is a confound; a `hall` seen on a terminal is evidence).
+ *   - `sub-venue/table.ts` — the emitted record schema plus the shipped seed vocabulary.
+ *   - `sub-venue/surfaces.ts` — phrase normalization, the phrase → record index, and the name-match
+ *     operator that gates the harvest.
+ *   - `sub-venue/wikidata.ts` — the designator-label SPARQL payload turned into surfaces.
+ *   - `sub-venue/head-nouns.ts` — the addressed form derived from an encyclopaedic label.
+ *   - `sub-venue/harvest.ts` — the harvestable row shape, its JSONL reader, and the attestation pass.
  *
  *   ── What `curated: false` means, and how a surface stops being it ────────────────────────────────
- *   Wikidata gives a CONCEPT NAME per language, not a designator as addressed. Q849706's Spanish label
- *   is `terminal aeroportuaria`; the addressed form is `Terminal`. Q240854 (`hall`) is `sala` in
- *   Italian, which names an ordinary room and would fire on half of Italy. So every machine-derived
- *   surface lands `curated: false`, and {@link SubVenueLexiconTable} consumers that gate parsing MUST
- *   filter to `curated: true`.
- *
- *   A surface becomes curated ONLY by matching a {@link SubVenuePromotion} in
- *   `sub-venue-promotions.ts` — a per-designator, per-LOCALE decision carrying the census that backs
- *   it. Promotion is per-locale because the same token is a designator in one language and a disaster
- *   in another: `hall` is `Halle 2` at Frankfurt and `Village Hall` at 3,205 British bus stops.
- *
- *   ── The seed DUPLICATES `neural/venue-structure.ts`, knowingly ───────────────────────────────────
- *   `@mailwoman/corpus` does not depend on `@mailwoman/neural` (the dependency runs the other way for
- *   the training path, and pulling onnxruntime into a corpus build to read three string arrays would be
- *   absurd), so the shipped vocabulary is re-declared below. That is a drift surface and it is stated
- *   rather than hidden: `sub-venue-lexicon.test.ts` pins the seed's contents literally, so a change in
- *   either place fails a test rather than passing silently.
+ *   Every machine-derived surface lands `curated: false`, and {@link SubVenueLexiconTable} consumers
+ *   that gate parsing MUST filter to `curated: true`. A surface becomes curated ONLY by matching a
+ *   {@link SubVenuePromotion} in `sub-venue-promotions.ts` — a per-designator, per-LOCALE decision
+ *   carrying the census that backs it. Promotion is per-locale because the same token is a designator in
+ *   one language and a disaster in another: `hall` is `Halle 2` at Frankfurt and `Village Hall` at 3,205
+ *   British bus stops.
  */
 import { readFileSync, statSync, writeFileSync } from "node:fs";
 import { basename, join } from "node:path";
 import { parseJSONStrict } from "@mailwoman/core/objects";
-import { TextSpliterator } from "spliterator";
 import { SUBVENUE_PROMOTIONS } from "./sub-venue-promotions.js";
-/**
- * This table's own data version. Bump when the source vintages or the build semantics change.
- *
- * `0.2.0` — wave 2: per-region attestation, the phrase-attribution fix (finding 3 above), derived head nouns, the
- * Overture source, and the `promotions[]` receipts section.
- */
-export const SUBVENUE_LEXICON_VERSION = "0.2.0";
-/**
- * Which side of the containment relation a designator names. Mirrors `@mailwoman/osm/sdk`'s `SubVenueTier`; re-declared
- * for the same dependency-direction reason as the seed.
- */
-export const LexiconTier = {
-    SubVenue: "subvenue",
-    Venue: "venue",
-};
-/**
- * The vocabulary that already ships in `neural/venue-structure.ts`, re-declared. See the module docstring for why this
- * duplication exists.
- *
- * `tier` is added here (the shipped list has no such field): the seven WOF placetypes plus `terminal`/`gate` are all
- * venue-INTERIOR, except `campus` and `building`, which name a whole venue as often as a part of one. They are marked
- * `subvenue` anyway, because that is the role the span proposer uses them in — `Building 43, Googleplex` is a unit
- * inside a venue.
- */
-export const SHIPPED_DESIGNATOR_SEED = [
-    { id: "arcade", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "building", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "campus", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "concourse", modifierEligible: true, provenance: ["wof:placetype"] },
-    { id: "enclosure", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "gate", modifierEligible: false, provenance: ["osm:aeroway=gate"] },
-    { id: "installation", modifierEligible: false, provenance: ["wof:placetype"] },
-    { id: "terminal", modifierEligible: true, provenance: ["osm:aeroway=terminal"] },
-    { id: "wing", modifierEligible: true, provenance: ["wof:placetype"] },
-];
-/**
- * The shipped positional modifiers, re-declared from `neural/venue-structure.ts`'s `VENUE_STRUCTURE_MODIFIERS`.
- */
-export const SHIPPED_MODIFIER_SEED = [
-    "central",
-    "east",
-    "front",
-    "inner",
-    "lower",
-    "main",
-    "north",
-    "outer",
-    "rear",
-    "south",
-    "upper",
-    "west",
-];
-/**
- * Designators the lexicon ADDS beyond what ships, each with the source that attests it.
- *
- * `platform`, `station` and `airport` come from the OSM extractor's rule table and are the rail/aviation venue-side
- * vocabulary the corpus line needs. `hall` and `satellite` come from Wikidata concepts and from
- * `wof-osm-placetype-map.mdx`'s own "plausible additions" note, which lists `hall` explicitly. `pier` joins them in
- * wave 2 on 282 Overture attestations in the `pier` category plus 162 in the GB extract — the corpus task names `Pier
- * C` as a target shape, so the record has to exist before a shard can generate it.
- *
- * None is `modifierEligible`: that claim needs a confound board per term AND per locale, and `sub-venue-promotions.ts`
- * is where those live. A promotion marks a SURFACE usable; it does not widen the modifier grammar.
- */
-export const PROPOSED_DESIGNATORS = [
-    { id: "airport", tier: LexiconTier.Venue, provenance: ["osm:aeroway=aerodrome"] },
-    { id: "hall", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q240854"] },
-    { id: "pier", tier: LexiconTier.SubVenue, provenance: ["overture:pier"] },
-    { id: "platform", tier: LexiconTier.SubVenue, provenance: ["osm:public_transport=platform", "osm:railway=platform"] },
-    { id: "satellite", tier: LexiconTier.SubVenue, provenance: ["wikidata:Q15990706"] },
-    { id: "station", tier: LexiconTier.Venue, provenance: ["osm:railway=station"] },
-];
-/**
- * `designatorID` → Wikidata QID, mirroring `fetch/wikidata-subvenue.ts`'s `SUBVENUE_CONCEPTS`. Re-declared here so the
- * builder stays a pure function over PARSED input rather than reaching into a fetch module for a constant; the test
- * pins the two against each other.
- */
-export const CONCEPT_QIDS = {
-    terminal: "Q849706",
-    gate: "Q247739",
-    concourse: "Q862212",
-    campus: "Q209465",
-    building: "Q41176",
-    arcade: "Q186637",
-    hall: "Q240854",
-    satellite: "Q15990706",
-};
-/**
- * How many real `ref` values each {@link IdentifierShape} keeps.
- *
- * Eight, not "all" and not one. The field exists so a shard author can see what a class actually CONTAINS — GB's
- * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
- * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
- * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
- */
-const IDENTIFIER_EXAMPLES_PER_SHAPE = 8;
-/**
- * Scripts whose case is meaningful to fold. Everything else is left as written — see {@link SubVenueSurface.phrase}.
- */
-const CASE_FOLDING_SCRIPT = /^[\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}\d\s\p{P}]+$/u;
-/**
- * Scripts written without spaces between words, where a token split cannot find a designator and a SUBSTRING test is
- * the correct operator. Han, Hiragana, Katakana; Hangul is excluded because Korean does space its words.
- *
- * The Germanic-compound argument that keeps {@link nameContainsSurface} token-bounded for Latin script does not transfer
- * here — there is no `-gate`/`-hall` street-name suffix class in Japanese, and `第1ターミナル` is unreachable by any token
- * split. Measured on the Japan extract: see the harvest counts in `corpus/data/PROVENANCE.md`.
- */
-const NON_SPACING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
-/**
- * Normalize a surface for the table: trim, collapse internal whitespace, and lowercase ONLY when the string is entirely
- * in a bicameral script. `ターミナルビル` and `航站楼` pass through untouched; `Flughafenterminal` folds.
- */
-export function normalizeSurface(text) {
-    const trimmed = text.trim().replaceAll(/\s+/gu, " ");
-    return CASE_FOLDING_SCRIPT.test(trimmed) ? trimmed.toLowerCase() : trimmed;
-}
-/**
- * Turn the Wikidata designator-label payload into surfaces.
- *
- * A row is dropped when its language tag is empty (an untagged literal, which Wikidata occasionally carries), when the
- * QID maps to no designator in {@link CONCEPT_QIDS}, or when the normalized phrase is empty. Everything that survives
- * lands `curated: false` — see the module docstring.
- */
-export function surfacesFromWikidata(payload, conceptQIDs = CONCEPT_QIDS) {
-    const byQID = new Map(Object.entries(conceptQIDs).map(([id, qid]) => [qid, id]));
-    const envelope = payload;
-    const seen = new Set();
-    const out = [];
-    for (const binding of envelope.results?.bindings ?? []) {
-        const qid = binding.item?.value?.split("/").pop();
-        const recordID = qid ? byQID.get(qid) : undefined;
-        const lang = binding.lang?.value;
-        const raw = binding.label?.value;
-        if (!recordID || !lang || !raw)
-            continue;
-        const phrase = normalizeSurface(raw);
-        if (!phrase)
-            continue;
-        const source = binding.kind?.value === "alt" ? "wikidata:alt" : "wikidata:label";
-        // A concept can carry the same string as both a label and an alias, and across dialect subtags
-        // (`zh`, `zh-cn`, `zh-hans` all say 航站楼). Key the dedupe on the tuple that identifies a row.
-        const key = `${phrase}${recordID}${lang}${source}`;
-        if (seen.has(key))
-            continue;
-        seen.add(key);
-        out.push({
-            phrase,
-            recordID,
-            recordKind: "designator",
-            lang,
-            region: "",
-            source,
-            curated: false,
-            observations: 0,
-            context: {},
-        });
-    }
-    return out;
-}
-/**
- * Classify an OSM `ref` into an {@link IdentifierShape} class.
- *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
- * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
- * more than one stand: `16-18` and `0/1`.
- */
-export function classifyIdentifier(ref) {
-    const value = ref.trim();
-    if (/^[0-9]+$/.test(value))
-        return "digit";
-    if (/^[A-Za-z]$/.test(value))
-        return "letter";
-    if (/^[A-Za-z]+[0-9]+$/.test(value))
-        return "letter-digit";
-    if (/^[0-9]+[A-Za-z]+$/.test(value))
-        return "digit-letter";
-    if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value))
-        return "range";
-    return "other";
-}
-/**
- * Index the surfaces accumulated so far by phrase. First writer wins, so a seed record beats a Wikidata alias that
- * happens to collide — `terminal` stays the `terminal` designator even though it is also an Italian alias for it.
- */
-export function buildSurfaceIndex(surfaces) {
-    const index = new Map();
-    for (const surface of surfaces) {
-        if (index.has(surface.phrase))
-            continue;
-        index.set(surface.phrase, { recordID: surface.recordID, recordKind: surface.recordKind });
-    }
-    return index;
-}
-/**
- * Every known phrase found in `name`, as whole-token runs for spacing scripts and as substrings for non-spacing ones.
- *
- * Token-boundary matching for Latin script, not substring: `Nordterminal` is a real German compound in which `terminal`
- * is a suffix, and a substring test would also fire on `Terminalstraße`. The compound case is a genuine miss and it is
- * the right miss — admitting suffix matches would fire on every `-hall`/`-gate` compound in Germanic and Nordic street
- * naming, which is exactly the confound class `Briggate`/`Kirkgate` represents.
- *
- * For Han/Kana names that rule finds nothing at all, because the script has no word boundaries: `第1ターミナル` splits into
- * one token that matches no surface. There the LONGEST known substring is the correct operator, and the compound
- * objection does not transfer — Japanese has no `-gate` street-name suffix class.
- */
-export function nameContainsSurfaces(name, index) {
-    const normalized = normalizeSurface(name);
-    const hits = new Set();
-    for (const token of normalized.split(/[\s,()/]+/u)) {
-        const stripped = token.replaceAll(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
-        if (stripped && index.has(stripped)) {
-            hits.add(stripped);
-        }
-    }
-    if (NON_SPACING_SCRIPT.test(normalized)) {
-        for (const [phrase] of index) {
-            if (NON_SPACING_SCRIPT.test(phrase) && normalized.includes(phrase)) {
-                hits.add(phrase);
-            }
-        }
-    }
-    return [...hits];
-}
-/**
- * Harvest attested phrases and identifier shapes out of a source's rows.
- *
- * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
- * is the whole reason this function is safe to run over raw OSM: see the module docstring's finding 1 for the Berlin
- * measurement that motivated it. The index also decides ATTRIBUTION (finding 3): a hit is a surface of the record the
- * PHRASE names, and the row's own designator is recorded as `context`.
- *
- * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
- */
-export function extractAttestedPhrases(rows, index, options = {}) {
-    const source = options.source ?? "osm";
-    const region = options.region ?? "";
-    /**
-     * `phrase\0lang\0source` → { count, context }.
-     */
-    const surfaceCounts = new Map();
-    /**
-     * `designatorID\0shape` → { count, examples }.
-     */
-    const shapes = new Map();
-    const note = (phrase, lang, sourceTag, context) => {
-        const key = `${phrase}${lang}${sourceTag}`;
-        const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map() };
-        entry.count++;
-        entry.context.set(context, (entry.context.get(context) ?? 0) + 1);
-        surfaceCounts.set(key, entry);
-    };
-    for (const row of rows) {
-        if (row.name) {
-            for (const hit of nameContainsSurfaces(row.name, index)) {
-                // `und` — the default `name` tag carries no language. Overture's `name` is the same: a
-                // primary name in whatever language the place uses, untagged.
-                note(hit, "und", `${source}:name`, row.designatorID);
-            }
-        }
-        for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
-            for (const hit of nameContainsSurfaces(localized, index)) {
-                note(hit, lang, `${source}:name:${lang}`, row.designatorID);
-            }
-        }
-        if (row.ref) {
-            const shape = classifyIdentifier(row.ref);
-            const key = `${row.designatorID}${shape}`;
-            const entry = shapes.get(key) ?? { count: 0, examples: new Set() };
-            entry.count++;
-            if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
-                entry.examples.add(row.ref.trim());
-            }
-            shapes.set(key, entry);
-        }
-    }
-    const surfaces = [...surfaceCounts].map(([key, entry]) => {
-        const [phrase, lang, sourceTag] = key.split("");
-        const record = index.get(phrase);
-        return {
-            phrase,
-            recordID: record.recordID,
-            recordKind: record.recordKind,
-            lang,
-            region,
-            source: sourceTag,
-            curated: false,
-            observations: entry.count,
-            context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
-        };
-    });
-    const identifierShapes = [...shapes].map(([key, entry]) => {
-        const [designatorID, shape] = key.split("");
-        return {
-            designatorID,
-            region,
-            shape,
-            observations: entry.count,
-            examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
-        };
-    });
-    return { surfaces, identifierShapes };
-}
-/**
- * Diacritic-flattened ASCII fold, for comparing a Slavic or Turkish inflection against its Latin root.
- */
-function asciiFold(text) {
-    return text
-        .normalize("NFD")
-        .replaceAll(/\p{Diacritic}/gu, "")
-        .toLowerCase();
-}
-/**
- * How many leading characters two ASCII-folded forms must share for one to count as the other's inflection.
- *
- * Five, or the id's own length when that is shorter (`hall`, `gate`, `wing`, `pier` are four). Measured against the
- * committed Wikidata pull: at five, `terminal`/`terminál`/`terminale`/`terminali`/`terminála`/`terminalo` are all
- * accepted for `terminal` while `campo` and `campws` are both rejected for `campus` (they share four). At six the
- * Spanish `satélite` is lost; at four, Italian `campo` is admitted and it means FIELD.
- */
-const HEAD_NOUN_PREFIX_FLOOR = 5;
-/**
- * The shortest substring a non-Latin head-noun candidate may be. Two: `航站` and `터미널` are both real, `楼` alone is
- * "building" and would fire on every Chinese building name.
- */
-const NON_LATIN_HEAD_MIN_LENGTH = 2;
-/**
- * How many head-noun candidates one non-Latin record+language group may contribute. Six — enough to carry `ターミナル`,
- * `ターミナルビル` and `旅客ターミナル` together, capped because the substring lattice of a nine-character label is large and, ranked
- * by attesting-surface count, nothing past the sixth has more than the minimum two.
- */
-const NON_LATIN_HEAD_CANDIDATE_CAP = 6;
-/**
- * Latin-script test — the scripts an ASCII-folded prefix comparison against a Latin designator id can work on.
- */
-const LATIN_PHRASE = /^[\p{Script=Latin}\d\s\p{P}]+$/u;
-/**
- * The scripts the shared-substring derivation is allowed to run on: Han, Hiragana, Katakana, Hangul.
- *
- * NARROWER than "not Latin", and the narrowing was earned. Run over every non-Latin phrase in the table, the derivation
- * produced 90 fragments of Cyrillic, Greek, Arabic, Thai, Burmese and Tamil words — `сгра`, `град`, `κτίρ`,
- * `ิ่งก่อสร้า` — because those languages have exactly one surface per concept and the only substrings shared inside a
- * group are pieces of one word. Every one of them was unusable, and none could ever be counted: `poi.db` is four
- * countries and this wave's extracts are GB, DE, FR, ES and JP, so nothing in reach attests a Thai or Burmese surface.
- * Deriving a candidate no available source can confirm is not a hypothesis, it is table weight.
- */
-const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
-/**
- * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
- * writes on an envelope.
- *
- * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
- * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
- * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
- * the head has to be extracted before the curation pass has anything to decide about.
- *
- * Two derivations, because the table holds two kinds of writing:
- *
- * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
- *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
- *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
- *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
- *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
- *   read "Ciudad Universitaria" as sub-venue structure.
- * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
- *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
- *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
- *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
- *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
- *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
- *   fragment.
- *
- * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
- * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
- * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
- * is, and a locale's own data is what confirms or kills it.
- */
-export function deriveHeadNounSurfaces(surfaces) {
-    const derived = new Map();
-    const seen = new Set(surfaces.map((s) => `${s.phrase}${s.recordID}${s.lang}`));
-    const emit = (phrase, from) => {
-        if (phrase === from.phrase)
-            return;
-        const key = `${phrase}${from.recordID}${from.lang}`;
-        if (seen.has(key) || derived.has(key))
-            return;
-        derived.set(key, {
-            phrase,
-            recordID: from.recordID,
-            recordKind: from.recordKind,
-            lang: from.lang,
-            region: "",
-            source: "derived:head-noun",
-            curated: false,
-            observations: 0,
-            context: {},
-        });
-    };
-    // ── Spacing scripts: prefix-match a token against a single-token surface of the same record ──────
-    // Latin script: a token that is a cognate of the designator's own canonical id.
-    for (const surface of surfaces) {
-        if (!LATIN_PHRASE.test(surface.phrase))
-            continue;
-        const parts = surface.phrase.split(/[^\p{L}\p{N}]+/u).filter(Boolean);
-        if (parts.length < 2)
-            continue;
-        const root = asciiFold(surface.recordID);
-        const floor = Math.min(HEAD_NOUN_PREFIX_FLOOR, root.length);
-        for (const part of parts) {
-            const folded = asciiFold(part);
-            if (folded.length >= floor && commonPrefixLength(folded, root) >= floor) {
-                emit(part, surface);
-            }
-        }
-    }
-    // Non-Latin script: substrings shared by two or more surfaces of the same record + language.
-    const groups = new Map();
-    for (const surface of surfaces) {
-        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
-            continue;
-        // Group `zh`, `zh-cn`, `zh-hant` together: they are writing systems for one vocabulary, and the
-        // simplified/traditional pair is exactly the evidence a shared substring needs.
-        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
-        const pool = groups.get(key) ?? new Set();
-        pool.add(surface.phrase);
-        groups.set(key, pool);
-    }
-    const candidatesByGroup = new Map();
-    for (const [key, pool] of groups) {
-        if (pool.size < 2)
-            continue;
-        candidatesByGroup.set(key, sharedSubstringCandidates(pool));
-    }
-    for (const surface of surfaces) {
-        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
-            continue;
-        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
-        for (const candidate of candidatesByGroup.get(key) ?? []) {
-            if (surface.phrase.includes(candidate)) {
-                emit(candidate, surface);
-            }
-        }
-    }
-    return [...derived.values()];
-}
-/**
- * Length of the shared leading run of two strings.
- */
-function commonPrefixLength(a, b) {
-    const limit = Math.min(a.length, b.length);
-    let i = 0;
-    while (i < limit && a[i] === b[i]) {
-        i++;
-    }
-    return i;
-}
-/**
- * Substrings occurring in at least two DISTINCT members of `pool`, ranked by that count and then by length, capped at
- * {@link NON_LATIN_HEAD_CANDIDATE_CAP}.
- *
- * A candidate never spans whitespace, and in a pool whose members contain whitespace a candidate must be a whole token
- * of some member. That is what keeps Korean `공항 터미널` from contributing a fragment straddling the space.
- *
- * MAXIMAL candidates only: one contained in a longer candidate carried by the SAME number of surfaces is dropped, since
- * counting can never separate the two. Every one of `ターミナル`'s five ja labels also contains `ターミ`, `ターミナ` and `ミナル`, so
- * without this the group contributes four indistinguishable candidates and the Japan harvest returns four identical
- * counts. `航站` survives next to `航站楼` because six surfaces carry it against that one's two.
- */
-function sharedSubstringCandidates(pool) {
-    const phrases = [...pool];
-    const spaced = phrases.some((phrase) => /\s/u.test(phrase));
-    const tokens = spaced ? new Set(phrases.flatMap((phrase) => phrase.split(/\s+/u).filter(Boolean))) : null;
-    const counts = new Map();
-    for (const phrase of phrases) {
-        const local = new Set();
-        for (let length = NON_LATIN_HEAD_MIN_LENGTH; length <= phrase.length; length++) {
-            for (let start = 0; start + length <= phrase.length; start++) {
-                const candidate = phrase.slice(start, start + length);
-                if (/\s/u.test(candidate))
-                    continue;
-                local.add(candidate);
-            }
-        }
-        for (const candidate of local) {
-            counts.set(candidate, (counts.get(candidate) ?? 0) + 1);
-        }
-    }
-    const kept = [...counts].filter(([candidate, count]) => count >= 2 && (!tokens || tokens.has(candidate)));
-    return kept
-        .filter(([candidate, count]) => kept.every(([other, otherCount]) => other === candidate || otherCount !== count || !other.includes(candidate)))
-        .toSorted((a, b) => b[1] - a[1] || b[0].length - a[0].length || a[0].localeCompare(b[0]))
-        .slice(0, NON_LATIN_HEAD_CANDIDATE_CAP)
-        .map(([candidate]) => candidate);
-}
+import { extractAttestedPhrases, readSubVenueJSONL } from "./sub-venue/harvest.js";
+import { deriveHeadNounSurfaces } from "./sub-venue/head-nouns.js";
+import { buildSurfaceIndex } from "./sub-venue/surfaces.js";
+import { CONCEPT_QIDS, LexiconTier, PROPOSED_DESIGNATORS, SHIPPED_DESIGNATOR_SEED, SHIPPED_MODIFIER_SEED, SUBVENUE_LEXICON_VERSION, } from "./sub-venue/table.js";
+import { surfacesFromWikidata } from "./sub-venue/wikidata.js";
+export * from "./sub-venue/harvest.js";
+export * from "./sub-venue/head-nouns.js";
+export * from "./sub-venue/surfaces.js";
+export * from "./sub-venue/table.js";
+export * from "./sub-venue/wikidata.js";
 /**
  * Apply the curation decisions to a surface list, IN PLACE on a copy.
  *
@@ -725,28 +212,6 @@ export function buildSubVenueLexicon(input) {
 export function serializeSubVenueLexicon(table) {
     return JSON.stringify(table, null, 2) + "\n";
 }
-/**
- * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
- * extract is a build output, and one malformed line should not cost the whole lexicon.
- */
-export function readSubVenueJSONL(path) {
-    const out = [];
-    // `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
-    // (52 MB for Great Britain), and materializing every segment before reading the first is exactly
-    // what the repo lint rule exists to prevent.
-    for (const line of TextSpliterator.from(readFileSync(path, "utf8"))) {
-        const trimmed = line.trim();
-        if (!trimmed)
-            continue;
-        try {
-            out.push(parseJSONStrict(trimmed));
-        }
-        catch {
-            continue;
-        }
-    }
-    return out;
-}
 /**
  * Read the fetch outputs, build the table, and write it.
  *