@mailwoman/corpus 9.0.0 → 9.2.0

package/out/src/tools/sub-venue/harvest.js

@@ -0,0 +1,168 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
+ *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
+ *   measured shape of each designator's identifier half.
+ *
+ *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
+ *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
+ *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
+ *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
+ *
+ *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
+ *   over raw OSM at all:
+ *
+ *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
+ *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
+ *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
+ *   designator token at all. So names are not harvested wholesale — a name contributes only when it
+ *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
+ *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
+ *
+ *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
+ *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
+ *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
+ *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
+ *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
+ *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
+ *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
+ */
+import { readFileSync } from "node:fs";
+import { parseJSONStrict } from "@mailwoman/core/objects";
+import { TextSpliterator } from "spliterator";
+import { nameContainsSurfaces } from "./surfaces.js";
+/**
+ * Classify an OSM `ref` into an {@link IdentifierShape} class.
+ *
+ * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
+ * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
+ * more than one stand: `16-18` and `0/1`.
+ */
+export function classifyIdentifier(ref) {
+    const value = ref.trim();
+    if (/^[0-9]+$/.test(value))
+        return "digit";
+    if (/^[A-Za-z]$/.test(value))
+        return "letter";
+    if (/^[A-Za-z]+[0-9]+$/.test(value))
+        return "letter-digit";
+    if (/^[0-9]+[A-Za-z]+$/.test(value))
+        return "digit-letter";
+    if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value))
+        return "range";
+    return "other";
+}
+/**
+ * How many real `ref` values each {@link IdentifierShape} keeps.
+ *
+ * Eight, not "all" and not one. The field exists so a shard author can see what a class actually CONTAINS — GB's
+ * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
+ * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
+ * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
+ */
+const IDENTIFIER_EXAMPLES_PER_SHAPE = 8;
+/**
+ * Harvest attested phrases and identifier shapes out of a source's rows.
+ *
+ * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
+ * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
+ * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
+ * own designator is recorded as `context`.
+ *
+ * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
+ */
+export function extractAttestedPhrases(rows, index, options = {}) {
+    const source = options.source ?? "osm";
+    const region = options.region ?? "";
+    /**
+     * `phrase\0lang\0source` → { count, context }.
+     */
+    const surfaceCounts = new Map();
+    /**
+     * `designatorID\0shape` → { count, examples }.
+     */
+    const shapes = new Map();
+    const note = (phrase, lang, sourceTag, context) => {
+        const key = `${phrase}\0${lang}\0${sourceTag}`;
+        const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map() };
+        entry.count++;
+        entry.context.set(context, (entry.context.get(context) ?? 0) + 1);
+        surfaceCounts.set(key, entry);
+    };
+    for (const row of rows) {
+        if (row.name) {
+            for (const hit of nameContainsSurfaces(row.name, index)) {
+                // `und` — the default `name` tag carries no language. Overture's `name` is the same: a
+                // primary name in whatever language the place uses, untagged.
+                note(hit, "und", `${source}:name`, row.designatorID);
+            }
+        }
+        for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
+            for (const hit of nameContainsSurfaces(localized, index)) {
+                note(hit, lang, `${source}:name:${lang}`, row.designatorID);
+            }
+        }
+        if (row.ref) {
+            const shape = classifyIdentifier(row.ref);
+            const key = `${row.designatorID}\0${shape}`;
+            const entry = shapes.get(key) ?? { count: 0, examples: new Set() };
+            entry.count++;
+            if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
+                entry.examples.add(row.ref.trim());
+            }
+            shapes.set(key, entry);
+        }
+    }
+    const surfaces = [...surfaceCounts].map(([key, entry]) => {
+        const [phrase, lang, sourceTag] = key.split("\0");
+        const record = index.get(phrase);
+        return {
+            phrase,
+            recordID: record.recordID,
+            recordKind: record.recordKind,
+            lang,
+            region,
+            source: sourceTag,
+            curated: false,
+            observations: entry.count,
+            context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
+        };
+    });
+    const identifierShapes = [...shapes].map(([key, entry]) => {
+        const [designatorID, shape] = key.split("\0");
+        return {
+            designatorID,
+            region,
+            shape,
+            observations: entry.count,
+            examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
+        };
+    });
+    return { surfaces, identifierShapes };
+}
+/**
+ * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
+ * extract is a build output, and one malformed line should not cost the whole lexicon.
+ */
+export function readSubVenueJSONL(path) {
+    const out = [];
+    // `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
+    // (52 MB for Great Britain), and materializing every segment before reading the first is exactly
+    // what the repo lint rule exists to prevent.
+    for (const line of TextSpliterator.from(readFileSync(path, "utf8"))) {
+        const trimmed = line.trim();
+        if (!trimmed)
+            continue;
+        try {
+            out.push(parseJSONStrict(trimmed));
+        }
+        catch {
+            continue;
+        }
+    }
+    return out;
+}
+//# sourceMappingURL=harvest.js.map

package/out/src/tools/sub-venue/harvest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"harvest.js","sourceRoot":"","sources":["../../../../src/tools/sub-venue/harvest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,oBAAoB,EAAqB,MAAM,eAAe,CAAA;AA2BvE;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAExB,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAA;IAE1C,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAE7C,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE1D,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE1D,IAAI,sCAAsC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAA;IAEtE,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,6BAA6B,GAAG,CAAC,CAAA;AAkBvC;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACrC,IAAkC,EAClC,KAAmB,EACnB,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAA;IACnC;;OAEG;IACH,MAAM,aAAa,GAAG,IAAI,GAAG,EAA2D,CAAA;IACxF;;OAEG;IACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoD,CAAA;IAE1E,MAAM,IAAI,GAAG,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAE,OAAe,EAAQ,EAAE;QACvF,MAAM,GAAG,GAAG,GAAG,MAAM,KAAK,IAAI,KAAK,SAAS,EAAE,CAAA;QAC9C,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAkB,EAAE,CAAA;QAExF,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACjE,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC,CAAA;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACzD,uFAAuF;gBACvF,8DAA8D;gBAC9D,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,EAAE,GAAG,CAAC,YAAY,CAAC,CAAA;YACrD,CAAC;QACF,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1E,KAAK,MAAM,GAAG,IAAI,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC1D,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACzC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,YAAY,KAAK,KAAK,EAAE,CAAA;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,CAAA;YAE1E,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,6BAA6B,EAAE,CAAC;gBACzD,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YACnC,CAAC;YAED,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAsB,CAAC,GAAG,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC3E,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAA6B,CAAA;QAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QAEjC,OAAO;YACN,MAAM;YACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,KAAK,CAAC,KAAK;YACzB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3G,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAsB,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5E,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAqB,CAAA;QAEjE,OAAO;YACN,YAAY;YACZ,MAAM;YACN,KAAK;YACL,YAAY,EAAE,KAAK,CAAC,KAAK;YACzB,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACpE,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAA;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,IAAY;IAC7C,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,8FAA8F;IAC9F,iGAAiG;IACjG,6CAA6C;IAC7C,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,EAAE,CAAC;QACrE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,CAAC;YACJ,GAAG,CAAC,IAAI,CAAC,eAAe,CAAqB,OAAO,CAAC,CAAC,CAAA;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}

package/out/src/tools/sub-venue/head-nouns.d.ts

@@ -0,0 +1,50 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Head-noun derivation for the sub-venue lexicon — proposing the ADDRESSED form of a designator
+ *   from the encyclopaedic label a vocabulary source carries.
+ *
+ *   Wikidata's label for a concept is the encyclopaedic name (`terminal aeroportuaria`, `letištní
+ *   terminál`, `havalimanı terminali`), while the addressed form is the bare head (`Terminal`,
+ *   `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so the head has to be
+ *   extracted before the curation pass has anything to decide about — that gap is why the first wave of
+ *   this table shipped 1,014 uncurated surfaces.
+ *
+ *   Everything derived lands `curated: false`. A derivation is a HYPOTHESIS about what the addressed
+ *   form is; a locale's own data is what confirms or kills it.
+ */
+import type { SubVenueSurface } from "./table.ts";
+/**
+ * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
+ * writes on an envelope.
+ *
+ * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
+ * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
+ * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
+ * the head has to be extracted before the curation pass has anything to decide about.
+ *
+ * Two derivations, because the table holds two kinds of writing:
+ *
+ * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
+ *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
+ *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
+ *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
+ *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
+ *   read "Ciudad Universitaria" as sub-venue structure.
+ * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
+ *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
+ *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
+ *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
+ *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
+ *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
+ *   fragment.
+ *
+ * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
+ * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
+ * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
+ * is, and a locale's own data is what confirms or kills it.
+ */
+export declare function deriveHeadNounSurfaces(surfaces: readonly SubVenueSurface[]): SubVenueSurface[];
+//# sourceMappingURL=head-nouns.d.ts.map

package/out/src/tools/sub-venue/head-nouns.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"head-nouns.d.ts","sourceRoot":"","sources":["../../../../src/tools/sub-venue/head-nouns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAoDjD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,GAAG,eAAe,EAAE,CA+E9F"}

package/out/src/tools/sub-venue/head-nouns.js

@@ -0,0 +1,210 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Head-noun derivation for the sub-venue lexicon — proposing the ADDRESSED form of a designator
+ *   from the encyclopaedic label a vocabulary source carries.
+ *
+ *   Wikidata's label for a concept is the encyclopaedic name (`terminal aeroportuaria`, `letištní
+ *   terminál`, `havalimanı terminali`), while the addressed form is the bare head (`Terminal`,
+ *   `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so the head has to be
+ *   extracted before the curation pass has anything to decide about — that gap is why the first wave of
+ *   this table shipped 1,014 uncurated surfaces.
+ *
+ *   Everything derived lands `curated: false`. A derivation is a HYPOTHESIS about what the addressed
+ *   form is; a locale's own data is what confirms or kills it.
+ */
+import { isPresent } from "@mailwoman/core/objects";
+/**
+ * Diacritic-flattened ASCII fold, for comparing a Slavic or Turkish inflection against its Latin root.
+ */
+function asciiFold(text) {
+    return text
+        .normalize("NFD")
+        .replaceAll(/\p{Diacritic}/gu, "")
+        .toLowerCase();
+}
+/**
+ * How many leading characters two ASCII-folded forms must share for one to count as the other's inflection.
+ *
+ * Five, or the id's own length when that is shorter (`hall`, `gate`, `wing`, `pier` are four). Measured against the
+ * committed Wikidata pull: at five, `terminal`/`terminál`/`terminale`/`terminali`/`terminála`/`terminalo` are all
+ * accepted for `terminal` while `campo` and `campws` are both rejected for `campus` (they share four). At six the
+ * Spanish `satélite` is lost; at four, Italian `campo` is admitted and it means FIELD.
+ */
+const HEAD_NOUN_PREFIX_FLOOR = 5;
+/**
+ * The shortest substring a non-Latin head-noun candidate may be. Two: `航站` and `터미널` are both real, `楼` alone is
+ * "building" and would fire on every Chinese building name.
+ */
+const NON_LATIN_HEAD_MIN_LENGTH = 2;
+/**
+ * How many head-noun candidates one non-Latin record+language group may contribute. Six — enough to carry `ターミナル`,
+ * `ターミナルビル` and `旅客ターミナル` together, capped because the substring lattice of a nine-character label is large and, ranked
+ * by attesting-surface count, nothing past the sixth has more than the minimum two.
+ */
+const NON_LATIN_HEAD_CANDIDATE_CAP = 6;
+/**
+ * Latin-script test — the scripts an ASCII-folded prefix comparison against a Latin designator id can work on.
+ */
+const LATIN_PHRASE = /^[\p{Script=Latin}\d\s\p{P}]+$/u;
+/**
+ * The scripts the shared-substring derivation is allowed to run on: Han, Hiragana, Katakana, Hangul.
+ *
+ * NARROWER than "not Latin", and the narrowing was earned. Run over every non-Latin phrase in the table, the derivation
+ * produced 90 fragments of Cyrillic, Greek, Arabic, Thai, Burmese and Tamil words — `сгра`, `град`, `κτίρ`,
+ * `ิ่งก่อสร้า` — because those languages have exactly one surface per concept and the only substrings shared inside a
+ * group are pieces of one word. Every one of them was unusable, and none could ever be counted: `poi.db` is four
+ * countries and this wave's extracts are GB, DE, FR, ES and JP, so nothing in reach attests a Thai or Burmese surface.
+ * Deriving a candidate no available source can confirm is not a hypothesis, it is table weight.
+ */
+const SHARED_SUBSTRING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}\p{Script=Hangul}]/u;
+/**
+ * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
+ * writes on an envelope.
+ *
+ * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
+ * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
+ * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
+ * the head has to be extracted before the curation pass has anything to decide about.
+ *
+ * Two derivations, because the table holds two kinds of writing:
+ *
+ * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
+ *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
+ *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
+ *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
+ *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
+ *   read "Ciudad Universitaria" as sub-venue structure.
+ * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
+ *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
+ *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
+ *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
+ *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
+ *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
+ *   fragment.
+ *
+ * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
+ * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
+ * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
+ * is, and a locale's own data is what confirms or kills it.
+ */
+export function deriveHeadNounSurfaces(surfaces) {
+    const derived = new Map();
+    const seen = new Set(surfaces.map((s) => `${s.phrase}\0${s.recordID}\0${s.lang}`));
+    const emit = (phrase, from) => {
+        if (phrase === from.phrase)
+            return;
+        const key = `${phrase}\0${from.recordID}\0${from.lang}`;
+        if (seen.has(key) || derived.has(key))
+            return;
+        derived.set(key, {
+            phrase,
+            recordID: from.recordID,
+            recordKind: from.recordKind,
+            lang: from.lang,
+            region: "",
+            source: "derived:head-noun",
+            curated: false,
+            observations: 0,
+            context: {},
+        });
+    };
+    // ── Spacing scripts: prefix-match a token against a single-token surface of the same record ──────
+    // Latin script: a token that is a cognate of the designator's own canonical id.
+    for (const surface of surfaces) {
+        if (!LATIN_PHRASE.test(surface.phrase))
+            continue;
+        const parts = surface.phrase.split(/[^\p{L}\p{N}]+/u).filter(isPresent);
+        if (parts.length < 2)
+            continue;
+        const root = asciiFold(surface.recordID);
+        const floor = Math.min(HEAD_NOUN_PREFIX_FLOOR, root.length);
+        for (const part of parts) {
+            const folded = asciiFold(part);
+            if (folded.length >= floor && commonPrefixLength(folded, root) >= floor) {
+                emit(part, surface);
+            }
+        }
+    }
+    // Non-Latin script: substrings shared by two or more surfaces of the same record + language.
+    const groups = new Map();
+    for (const surface of surfaces) {
+        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
+            continue;
+        // Group `zh`, `zh-cn`, `zh-hant` together: they are writing systems for one vocabulary, and the
+        // simplified/traditional pair is exactly the evidence a shared substring needs.
+        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
+        const pool = groups.get(key) ?? new Set();
+        pool.add(surface.phrase);
+        groups.set(key, pool);
+    }
+    const candidatesByGroup = new Map();
+    for (const [key, pool] of groups) {
+        if (pool.size < 2)
+            continue;
+        candidatesByGroup.set(key, sharedSubstringCandidates(pool));
+    }
+    for (const surface of surfaces) {
+        if (!SHARED_SUBSTRING_SCRIPT.test(surface.phrase))
+            continue;
+        const key = `${surface.recordID} ${surface.lang.split(/[-_]/u)[0]}`;
+        for (const candidate of candidatesByGroup.get(key) ?? []) {
+            if (surface.phrase.includes(candidate)) {
+                emit(candidate, surface);
+            }
+        }
+    }
+    return [...derived.values()];
+}
+/**
+ * Length of the shared leading run of two strings.
+ */
+function commonPrefixLength(a, b) {
+    const limit = Math.min(a.length, b.length);
+    let i = 0;
+    while (i < limit && a[i] === b[i]) {
+        i++;
+    }
+    return i;
+}
+/**
+ * Substrings occurring in at least two DISTINCT members of `pool`, ranked by that count and then by length, capped at
+ * {@link NON_LATIN_HEAD_CANDIDATE_CAP}.
+ *
+ * A candidate never spans whitespace, and in a pool whose members contain whitespace a candidate must be a whole token
+ * of some member. That is what keeps Korean `공항 터미널` from contributing a fragment straddling the space.
+ *
+ * MAXIMAL candidates only: one contained in a longer candidate carried by the SAME number of surfaces is dropped, since
+ * counting can never separate the two. Every one of `ターミナル`'s five ja labels also contains `ターミ`, `ターミナ` and `ミナル`, so
+ * without this the group contributes four indistinguishable candidates and the Japan harvest returns four identical
+ * counts. `航站` survives next to `航站楼` because six surfaces carry it against that one's two.
+ */
+function sharedSubstringCandidates(pool) {
+    const phrases = [...pool];
+    const spaced = phrases.some((phrase) => /\s/u.test(phrase));
+    const tokens = spaced ? new Set(phrases.flatMap((phrase) => phrase.split(/\s+/u).filter(isPresent))) : null;
+    const counts = new Map();
+    for (const phrase of phrases) {
+        const local = new Set();
+        for (let length = NON_LATIN_HEAD_MIN_LENGTH; length <= phrase.length; length++) {
+            for (let start = 0; start + length <= phrase.length; start++) {
+                const candidate = phrase.slice(start, start + length);
+                if (/\s/u.test(candidate))
+                    continue;
+                local.add(candidate);
+            }
+        }
+        for (const candidate of local) {
+            counts.set(candidate, (counts.get(candidate) ?? 0) + 1);
+        }
+    }
+    const kept = [...counts].filter(([candidate, count]) => count >= 2 && (!tokens || tokens.has(candidate)));
+    return kept
+        .filter(([candidate, count]) => kept.every(([other, otherCount]) => other === candidate || otherCount !== count || !other.includes(candidate)))
+        .toSorted((a, b) => b[1] - a[1] || b[0].length - a[0].length || a[0].localeCompare(b[0]))
+        .slice(0, NON_LATIN_HEAD_CANDIDATE_CAP)
+        .map(([candidate]) => candidate);
+}
+//# sourceMappingURL=head-nouns.js.map

package/out/src/tools/sub-venue/head-nouns.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"head-nouns.js","sourceRoot":"","sources":["../../../../src/tools/sub-venue/head-nouns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AAInD;;GAEG;AACH,SAAS,SAAS,CAAC,IAAY;IAC9B,OAAO,IAAI;SACT,SAAS,CAAC,KAAK,CAAC;SAChB,UAAU,CAAC,iBAAiB,EAAE,EAAE,CAAC;SACjC,WAAW,EAAE,CAAA;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEhC;;;GAGG;AACH,MAAM,yBAAyB,GAAG,CAAC,CAAA;AAEnC;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC;;GAEG;AACH,MAAM,YAAY,GAAG,iCAAiC,CAAA;AAEtD;;;;;;;;;GASG;AACH,MAAM,uBAAuB,GAAG,0EAA0E,CAAA;AAE1G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAoC;IAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAElF,MAAM,IAAI,GAAG,CAAC,MAAc,EAAE,IAAqB,EAAQ,EAAE;QAC5D,IAAI,MAAM,KAAK,IAAI,CAAC,MAAM;YAAE,OAAM;QAElC,MAAM,GAAG,GAAG,GAAG,MAAM,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,IAAI,EAAE,CAAA;QAEvD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAM;QAE7C,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;YAChB,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,mBAAmB;YAC3B,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,EAAE;SACX,CAAC,CAAA;IACH,CAAC,CAAA;IAED,oGAAoG;IACpG,gFAAgF;IAChF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAEhD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAEvE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAQ;QAE9B,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;YAE9B,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;gBACzE,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACpB,CAAC;QACF,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAA;IAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE3D,gGAAgG;QAChG,gFAAgF;QAChF,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,EAAE,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAU,CAAA;QACjD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IACtB,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAoB,CAAA;IAErD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,SAAQ;QAC3B,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE3D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,EAAE,CAAA;QAEpE,KAAK,MAAM,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YACzB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAA;IAC1C,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,yBAAyB,CAAC,IAAyB;IAC3D,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACzB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3G,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;QAE/B,KAAK,IAAI,MAAM,GAAG,yBAAyB,EAAE,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;YAChF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;gBAErD,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;oBAAE,SAAQ;gBACnC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IAEzG,OAAO,IAAI;SACT,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAC9B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,UAAU,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAC9G;SACA,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxF,KAAK,CAAC,CAAC,EAAE,4BAA4B,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAA;AAClC,CAAC"}

package/out/src/tools/sub-venue/surfaces.d.ts

@@ -0,0 +1,45 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Surface-form primitives for the sub-venue lexicon: the normalizer every phrase passes
+ *   through, the phrase → record index built over the surfaces accumulated so far, and the operator
+ *   that decides whether a feature's name contains a known designator.
+ *
+ *   The two matching rules here are script-conditional, and both narrowings are load-bearing — see
+ *   {@link NON_SPACING_SCRIPT} and {@link nameContainsSurfaces}.
+ */
+import type { SubVenueSurface } from "./table.ts";
+/**
+ * Normalize a surface for the table: trim, collapse internal whitespace, and lowercase ONLY when the string is entirely
+ * in a bicameral script. `ターミナルビル` and `航站楼` pass through untouched; `Flughafenterminal` folds.
+ */
+export declare function normalizeSurface(text: string): string;
+/**
+ * A phrase → record index, keyed on the normalized phrase. Built by {@link buildSurfaceIndex} from the surfaces present
+ * before the harvest runs, and the reason a matched phrase can be attributed to the record it actually names.
+ */
+export type SurfaceIndex = ReadonlyMap<string, {
+    recordID: string;
+    recordKind: "designator" | "modifier";
+}>;
+/**
+ * Index the surfaces accumulated so far by phrase. First writer wins, so a seed record beats a Wikidata alias that
+ * happens to collide — `terminal` stays the `terminal` designator even though it is also an Italian alias for it.
+ */
+export declare function buildSurfaceIndex(surfaces: readonly SubVenueSurface[]): SurfaceIndex;
+/**
+ * Every known phrase found in `name`, as whole-token runs for spacing scripts and as substrings for non-spacing ones.
+ *
+ * Token-boundary matching for Latin script, not substring: `Nordterminal` is a real German compound in which `terminal`
+ * is a suffix, and a substring test would also fire on `Terminalstraße`. The compound case is a genuine miss and it is
+ * the right miss — admitting suffix matches would fire on every `-hall`/`-gate` compound in Germanic and Nordic street
+ * naming, which is exactly the confound class `Briggate`/`Kirkgate` represents.
+ *
+ * For Han/Kana names that rule finds nothing at all, because the script has no word boundaries: `第1ターミナル` splits into
+ * one token that matches no surface. There the LONGEST known substring is the correct operator, and the compound
+ * objection does not transfer — Japanese has no `-gate` street-name suffix class.
+ */
+export declare function nameContainsSurfaces(name: string, index: SurfaceIndex): string[];
+//# sourceMappingURL=surfaces.d.ts.map

package/out/src/tools/sub-venue/surfaces.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"surfaces.d.ts","sourceRoot":"","sources":["../../../../src/tools/sub-venue/surfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAA;AAiBjD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIrD;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,YAAY,GAAG,UAAU,CAAA;CAAE,CAAC,CAAA;AAE3G;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,GAAG,YAAY,CASpF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,MAAM,EAAE,CAqBhF"}

package/out/src/tools/sub-venue/surfaces.js

@@ -0,0 +1,77 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   @file Surface-form primitives for the sub-venue lexicon: the normalizer every phrase passes
+ *   through, the phrase → record index built over the surfaces accumulated so far, and the operator
+ *   that decides whether a feature's name contains a known designator.
+ *
+ *   The two matching rules here are script-conditional, and both narrowings are load-bearing — see
+ *   {@link NON_SPACING_SCRIPT} and {@link nameContainsSurfaces}.
+ */
+/**
+ * Scripts whose case is meaningful to fold. Everything else is left as written — see {@link SubVenueSurface.phrase}.
+ */
+const CASE_FOLDING_SCRIPT = /^[\p{Script=Latin}\p{Script=Cyrillic}\p{Script=Greek}\d\s\p{P}]+$/u;
+/**
+ * Scripts written without spaces between words, where a token split cannot find a designator and a SUBSTRING test is
+ * the correct operator. Han, Hiragana, Katakana; Hangul is excluded because Korean does space its words.
+ *
+ * The Germanic-compound argument that keeps {@link nameContainsSurfaces} token-bounded for Latin script does not
+ * transfer here — there is no `-gate`/`-hall` street-name suffix class in Japanese, and `第1ターミナル` is unreachable by any
+ * token split. Measured on the Japan extract: see the harvest counts in `corpus/data/PROVENANCE.md`.
+ */
+const NON_SPACING_SCRIPT = /[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]/u;
+/**
+ * Normalize a surface for the table: trim, collapse internal whitespace, and lowercase ONLY when the string is entirely
+ * in a bicameral script. `ターミナルビル` and `航站楼` pass through untouched; `Flughafenterminal` folds.
+ */
+export function normalizeSurface(text) {
+    const trimmed = text.trim().replaceAll(/\s+/gu, " ");
+    return CASE_FOLDING_SCRIPT.test(trimmed) ? trimmed.toLowerCase() : trimmed;
+}
+/**
+ * Index the surfaces accumulated so far by phrase. First writer wins, so a seed record beats a Wikidata alias that
+ * happens to collide — `terminal` stays the `terminal` designator even though it is also an Italian alias for it.
+ */
+export function buildSurfaceIndex(surfaces) {
+    const index = new Map();
+    for (const surface of surfaces) {
+        if (index.has(surface.phrase))
+            continue;
+        index.set(surface.phrase, { recordID: surface.recordID, recordKind: surface.recordKind });
+    }
+    return index;
+}
+/**
+ * Every known phrase found in `name`, as whole-token runs for spacing scripts and as substrings for non-spacing ones.
+ *
+ * Token-boundary matching for Latin script, not substring: `Nordterminal` is a real German compound in which `terminal`
+ * is a suffix, and a substring test would also fire on `Terminalstraße`. The compound case is a genuine miss and it is
+ * the right miss — admitting suffix matches would fire on every `-hall`/`-gate` compound in Germanic and Nordic street
+ * naming, which is exactly the confound class `Briggate`/`Kirkgate` represents.
+ *
+ * For Han/Kana names that rule finds nothing at all, because the script has no word boundaries: `第1ターミナル` splits into
+ * one token that matches no surface. There the LONGEST known substring is the correct operator, and the compound
+ * objection does not transfer — Japanese has no `-gate` street-name suffix class.
+ */
+export function nameContainsSurfaces(name, index) {
+    const normalized = normalizeSurface(name);
+    const hits = new Set();
+    for (const token of normalized.split(/[\s,()/]+/u)) {
+        const stripped = token.replaceAll(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, "");
+        if (stripped && index.has(stripped)) {
+            hits.add(stripped);
+        }
+    }
+    if (NON_SPACING_SCRIPT.test(normalized)) {
+        for (const [phrase] of index) {
+            if (NON_SPACING_SCRIPT.test(phrase) && normalized.includes(phrase)) {
+                hits.add(phrase);
+            }
+        }
+    }
+    return [...hits];
+}
+//# sourceMappingURL=surfaces.js.map

package/out/src/tools/sub-venue/surfaces.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"surfaces.js","sourceRoot":"","sources":["../../../../src/tools/sub-venue/surfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH;;GAEG;AACH,MAAM,mBAAmB,GAAG,oEAAoE,CAAA;AAEhG;;;;;;;GAOG;AACH,MAAM,kBAAkB,GAAG,yDAAyD,CAAA;AAEpF;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;IAEpD,OAAO,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAA;AAC3E,CAAC;AAQD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAoC;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuE,CAAA;IAE5F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QACvC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAmB;IACrE,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAA;QAE1E,IAAI,QAAQ,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACnB,CAAC;IACF,CAAC;IAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YAC9B,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACjB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;AACjB,CAAC"}