@mailwoman/corpus 9.3.0 → 10.0.0

package/out/tools/slice-translit.js

@@ -1,166 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Build per-script parquet slices from the DeepSeek-generated transliteration JSONL and emit the
- *   corpus-v0.4.0 MANIFEST that combines them with the existing kryptonite + v0.3.0 slices.
- *
- *   Sibling to `slice-kryptonite.ts`. The two modules share the same composition pattern: take a
- *   base MANIFEST, append new slices, write a combined MANIFEST. Differences specific to
- *   transliteration:
- *
- *   - One JSONL contains rows from N target scripts (source = `deepseek-translit-<slug>`). We bucket by
- *       `source` and write one slice per script so `audit.ts` can attribute each slice to its
- *       synthetic source without relying on filename-prefix inference.
- *   - Each slice is written to `train/part-translit-<slug>.parquet` (distinct from kryptonite's
- *       `part-0000.parquet`, which v0.4.0's first builder already produced).
- *   - Inherits the path-canonicalization fix flagged in Thread B's postmortem: v0.3.0 slice paths are
- *       rewritten from `$MAILWOMAN_DATA_ROOT/...` to `/data/...` in the combined MANIFEST so
- *       all paths share one container-friendly form.
- *
- *   See docs/engineering/reference/CORPUS_V0_4_0_GENERATION.md for prompts, model, and the
- *   reproducibility contract.
- *
- *   Invoke via `mailwoman corpus slice translit \
- *   --jsonl /data/corpus/versioned/v0.4.0/transliteration/canonical-transliteration.jsonl \
- *   --base-manifest /data/corpus/versioned/v0.4.0/corpus-v0.4.0/MANIFEST.json \
- *   --out-dir /data/corpus/versioned/v0.4.0`
- */
-import { mailwomanDataRoot } from "@mailwoman/core/data-root";
-import { pathExists, readLocalJSONFile, tryStat } from "@mailwoman/core/fs/readers";
-import { writeLocalJSONFile, writeLocalTextFile, makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { join } from "path-ts";
-import { JSONSpliterator } from "spliterator";
-import { alignRow, PARQUET_COLUMNS, ROW_GROUP_SIZE, rowToParquet, SLICE_COMPRESSION, writeParquetRows } from "#utils";
-function toCanonicalRow(raw, corpusVersion) {
-    return {
-        raw: raw["raw"],
-        components: raw["components"],
-        country: raw["country"] ?? "US",
-        locale: raw["locale"] ?? undefined,
-        source: raw["source"],
-        source_id: raw["source_id"],
-        corpus_version: corpusVersion,
-        license: raw["license"] ?? "Synthetic (DeepSeek-v4-flash, AGPL-compatible)",
-        synth: raw["synth"],
-    };
-}
-/**
- * Write one slice for a single source slug. Returns the populated SliceDescriptor + a list of quarantine reasons for
- * rows that failed alignment.
- */
-async function writeOneSlice(rows, outPath, source) {
-    let firstSourceID = "";
-    let lastSourceID = "";
-    for (const row of rows) {
-        if (firstSourceID === "") {
-            firstSourceID = row.source_id;
-        }
-        lastSourceID = row.source_id;
-    }
-    await writeParquetRows(rows.map(rowToParquet), outPath);
-    const fileStat = await tryStat(outPath);
-    const sha256 = await sha256File(outPath);
-    return {
-        split: "train",
-        path: outPath,
-        format: "parquet",
-        compression: SLICE_COMPRESSION,
-        rows: rows.length,
-        bytes: fileStat?.size ?? 0,
-        sha256,
-        first_source_id: firstSourceID,
-        last_source_id: lastSourceID,
-        // Stamp source so audit.ts attributes the slice without falling back to filename-prefix inference.
-        source,
-    };
-}
-function canonicalizeSlicePath(path, legacyPrefix, canonicalPrefix) {
-    if (path.startsWith(legacyPrefix))
-        return canonicalPrefix + path.slice(legacyPrefix.length);
-    return path;
-}
-export async function buildTranslitSlice(options, report) {
-    const corpusVersion = options.corpusVersion ?? "0.4.0";
-    // This is a portable manifest namespace, not a host filesystem default.
-    const canonicalPathPrefix = options.canonicalPathPrefix ?? "/data/";
-    const legacyPathPrefix = options.legacyPathPrefix ?? `${mailwomanDataRoot()}/`;
-    if (!(await pathExists(options.jsonl)))
-        throw new Error(`jsonl not found: ${options.jsonl}`);
-    if (!(await pathExists(options.baseManifest)))
-        throw new Error(`base-manifest not found: ${options.baseManifest}`);
-    const corpusDir = join(options.outDir, `corpus-v${corpusVersion}`);
-    const trainDir = join(corpusDir, "train");
-    await makeDirectories(trainDir);
-    // Bucket canonical rows by source. Quarantined rows are logged.
-    const buckets = new Map();
-    const quarantine = [];
-    let totalIn = 0;
-    for await (const raw of JSONSpliterator.fromAsync(options.jsonl)) {
-        totalIn++;
-        const canon = toCanonicalRow(raw, corpusVersion);
-        const result = alignRow(canon);
-        if (result.kind !== "labeled") {
-            quarantine.push(`${canon.source_id}\t${result.row.reason}`);
-            continue;
-        }
-        const bucket = buckets.get(canon.source);
-        if (bucket) {
-            bucket.push(result.row);
-        }
-        else {
-            buckets.set(canon.source, [result.row]);
-        }
-    }
-    report?.(`read ${totalIn} rows; ${quarantine.length} quarantined; ${buckets.size} script buckets`);
-    const newSlices = [];
-    const sortedKeys = [...buckets.keys()].toSorted();
-    for (const source of sortedKeys) {
-        const rows = buckets.get(source);
-        const slug = source.startsWith("deepseek-translit-") ? source.slice("deepseek-translit-".length) : source;
-        const outPath = join(trainDir, `part-translit-${slug}.parquet`);
-        const descriptor = await writeOneSlice(rows, outPath, source);
-        newSlices.push(descriptor);
-        report?.(`  ${source}: ${descriptor.rows} rows → ${outPath} (${descriptor.bytes} bytes)`);
-    }
-    if (quarantine.length) {
-        const qPath = join(corpusDir, "quarantine-transliteration.tsv");
-        await writeLocalTextFile(quarantine.join("\n") + "\n", qPath);
-        report?.(`quarantine log → ${qPath} (${quarantine.length} rows)`);
-    }
-    // Compose final MANIFEST: rewrite base.slices paths from /mnt/playpen/... → /data/... and append
-    // the new translit slices. Kryptonite slice already lives in the base manifest (it was written
-    // there by Thread B).
-    const base = await readLocalJSONFile(options.baseManifest);
-    const rewrittenBase = base.slices.map((sh) => ({
-        ...sh,
-        path: canonicalizeSlicePath(sh.path, legacyPathPrefix, canonicalPathPrefix),
-    }));
-    const newTrainRows = newSlices.reduce((sum, sh) => sum + sh.rows, 0);
-    const combined = {
-        corpus_version: corpusVersion,
-        schema: PARQUET_COLUMNS,
-        rows_per_slice: base.rows_per_slice,
-        row_group_size: base.row_group_size ?? ROW_GROUP_SIZE,
-        slices: [...rewrittenBase, ...newSlices],
-        counts: {
-            train: base.counts.train + newTrainRows,
-            val: base.counts.val,
-            test: base.counts.test,
-        },
-        total_rows: base.total_rows + newTrainRows,
-    };
-    const combinedPath = join(corpusDir, "MANIFEST.json");
-    await writeLocalJSONFile(combined, combinedPath);
-    report?.(`wrote combined manifest → ${combinedPath}`);
-    report?.(`  total_rows=${combined.total_rows} (base=${base.total_rows}, added=${newTrainRows})`);
-    report?.(`  slices=${combined.slices.length} (base=${base.slices.length}, added=${newSlices.length})`);
-    report?.(`  compression=${SLICE_COMPRESSION}`);
-    const pathFix = rewrittenBase.filter((s, i) => s.path !== base.slices[i].path).length;
-    if (pathFix > 0) {
-        report?.(`  path-canonicalized base slices: ${pathFix} (legacy '${legacyPathPrefix}' → '${canonicalPathPrefix}')`);
-    }
-}
-//# sourceMappingURL=slice-translit.js.map

package/out/tools/slice-translit.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"slice-translit.js","sourceRoot":"","sources":["../../lib/tools/slice-translit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACnF,OAAO,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AACpG,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAG7C,OAAO,EAAE,QAAQ,EAAE,eAAe,EAAE,cAAc,EAAE,YAAY,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,MAAM,QAAQ,CAAA;AAwBrH,SAAS,cAAc,CAAC,GAA4B,EAAE,aAAqB;IAC1E,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,KAAK,CAAW;QACzB,UAAU,EAAE,GAAG,CAAC,YAAY,CAA2B;QACvD,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,IAAI;QAC3C,MAAM,EAAG,GAAG,CAAC,QAAQ,CAAY,IAAI,SAAS;QAC9C,MAAM,EAAE,GAAG,CAAC,QAAQ,CAAW;QAC/B,SAAS,EAAE,GAAG,CAAC,WAAW,CAAW;QACrC,cAAc,EAAE,aAAa;QAC7B,OAAO,EAAG,GAAG,CAAC,SAAS,CAAY,IAAI,gDAAgD;QACvF,KAAK,EAAE,GAAG,CAAC,OAAO,CAA0B;KAC5C,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,aAAa,CAAC,IAA2B,EAAE,OAAe,EAAE,MAAc;IACxF,IAAI,aAAa,GAAG,EAAE,CAAA;IACtB,IAAI,YAAY,GAAG,EAAE,CAAA;IAErB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,aAAa,KAAK,EAAE,EAAE,CAAC;YAC1B,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;QAC9B,CAAC;QAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;IAC7B,CAAC;IAED,MAAM,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,OAAO,CAAC,CAAA;IAEvD,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAA;IACvC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,OAAO;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;QAC1B,MAAM;QACN,eAAe,EAAE,aAAa;QAC9B,cAAc,EAAE,YAAY;QAC5B,mGAAmG;QACnG,MAAM;KACN,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,YAAoB,EAAE,eAAuB;IACzF,IAAI,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;IAE3F,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAA;IACtD,wEAAwE;IACxE,MAAM,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,QAAQ,CAAA;IACnE,MAAM,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,GAAG,iBAAiB,EAAE,GAAG,CAAA;IAE9E,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,oBAAoB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAE5F,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,OAAO,CAAC,YAAY,EAAE,CAAC,CAAA;IAElH,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,WAAW,aAAa,EAAE,CAAC,CAAA;IAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IACzC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;IAE/B,gEAAgE;IAChE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAwB,CAAA;IAC/C,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,eAAe,CAAC,SAAS,CAA0B,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3F,OAAO,EAAE,CAAA;QACT,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,aAAa,CAAC,CAAA;QAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,UAAU,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;YAE3D,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QAExC,IAAI,MAAM,EAAE,CAAC;YACZ,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAA;QACxC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,QAAQ,OAAO,UAAU,UAAU,CAAC,MAAM,iBAAiB,OAAO,CAAC,IAAI,iBAAiB,CAAC,CAAA;IAElG,MAAM,SAAS,GAAsB,EAAE,CAAA;IACvC,MAAM,UAAU,GAAG,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEjD,KAAK,MAAM,MAAM,IAAI,UAAU,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QACjC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACzG,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,EAAE,iBAAiB,IAAI,UAAU,CAAC,CAAA;QAC/D,MAAM,UAAU,GAAG,MAAM,aAAa,CAAC,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAC7D,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;QAC1B,MAAM,EAAE,CAAC,KAAK,MAAM,KAAK,UAAU,CAAC,IAAI,WAAW,OAAO,KAAK,UAAU,CAAC,KAAK,SAAS,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,gCAAgC,CAAC,CAAA;QAC/D,MAAM,kBAAkB,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE,KAAK,CAAC,CAAA;QAC7D,MAAM,EAAE,CAAC,oBAAoB,KAAK,KAAK,UAAU,CAAC,MAAM,QAAQ,CAAC,CAAA;IAClE,CAAC;IAED,iGAAiG;IACjG,+FAA+F;IAC/F,sBAAsB;IACtB,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAgB,OAAO,CAAC,YAAY,CAAC,CAAA;IAEzE,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;QAC9C,GAAG,EAAE;QACL,IAAI,EAAE,qBAAqB,CAAC,EAAE,CAAC,IAAI,EAAE,gBAAgB,EAAE,mBAAmB,CAAC;KAC3E,CAAC,CAAC,CAAA;IAEH,MAAM,YAAY,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAEpE,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,aAAa;QAC7B,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,cAAc,EAAE,IAAI,CAAC,cAAc,IAAI,cAAc;QACrD,MAAM,EAAE,CAAC,GAAG,aAAa,EAAE,GAAG,SAAS,CAAC;QACxC,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,YAAY;YACvC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,YAAY;KAC1C,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACrD,MAAM,kBAAkB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAA;IAChD,MAAM,EAAE,CAAC,6BAA6B,YAAY,EAAE,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,gBAAgB,QAAQ,CAAC,UAAU,UAAU,IAAI,CAAC,UAAU,WAAW,YAAY,GAAG,CAAC,CAAA;IAChG,MAAM,EAAE,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,MAAM,UAAU,IAAI,CAAC,MAAM,CAAC,MAAM,WAAW,SAAS,CAAC,MAAM,GAAG,CAAC,CAAA;IACtG,MAAM,EAAE,CAAC,iBAAiB,iBAAiB,EAAE,CAAC,CAAA;IAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,IAAI,CAAC,CAAC,MAAM,CAAA;IAEtF,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;QACjB,MAAM,EAAE,CAAC,qCAAqC,OAAO,aAAa,gBAAgB,QAAQ,mBAAmB,IAAI,CAAC,CAAA;IACnH,CAAC;AACF,CAAC"}

package/out/tools/sub-venue/harvest.d.ts

@@ -1,100 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
- *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
- *   measured shape of each designator's identifier half.
- *
- *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
- *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
- *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
- *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- *
- *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
- *   over raw OSM at all:
- *
- *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
- *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
- *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
- *   designator token at all. So names are not harvested wholesale — a name contributes only when it
- *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
- *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
- *
- *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
- *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
- *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
- *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
- *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
- *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
- *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
- */
-import { type SurfaceIndex } from "#tools/sub-venue/surfaces";
-import type { IdentifierShape, SubVenueSurface } from "#tools/sub-venue/table";
-/**
- * One row of a harvestable source, as read back off JSONL or out of a layer database.
- *
- * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
- * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
- * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
- * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- */
-export interface SubVenueHarvestRow {
-    /**
-     * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
-     * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
-     */
-    designatorID: string;
-    /**
-     * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
-     * read it; the sub-venue slice recipe fills its venue slot from it.
-     */
-    tier?: string;
-    name?: string | null;
-    ref?: string | null;
-    localizedNames?: Record<string, string>;
-}
-/**
- * Classify an OSM `ref` into an {@link IdentifierShape} class.
- *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
- * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
- * more than one stand: `16-18` and `0/1`.
- */
-export declare function classifyIdentifier(ref: string): string;
-/**
- * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
- *
- * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
- */
-export interface HarvestOptions {
-    /**
-     * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
-     */
-    source?: string;
-    /**
-     * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
-     */
-    region?: string;
-}
-/**
- * Harvest attested phrases and identifier shapes out of a source's rows.
- *
- * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
- * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
- * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
- * own designator is recorded as `context`.
- *
- * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
- */
-export declare function extractAttestedPhrases(rows: Iterable<SubVenueHarvestRow>, index: SurfaceIndex, options?: HarvestOptions): {
-    surfaces: SubVenueSurface[];
-    identifierShapes: IdentifierShape[];
-};
-/**
- * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
- * extract is a build output, and one malformed line should not cost the whole lexicon.
- */
-export declare function readSubVenueJSONL(path: string): Promise<SubVenueHarvestRow[]>;
-//# sourceMappingURL=harvest.d.ts.map

package/out/tools/sub-venue/harvest.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"harvest.d.ts","sourceRoot":"","sources":["../../../lib/tools/sub-venue/harvest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAMH,OAAO,EAAwB,KAAK,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACnF,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAE9E;;;;;;;GAOG;AACH,MAAM,WAAW,kBAAkB;IAClC;;;OAGG;IACH,YAAY,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,GAAG,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACvC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CActD;AAYD;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACrC,IAAI,EAAE,QAAQ,CAAC,kBAAkB,CAAC,EAClC,KAAK,EAAE,YAAY,EACnB,OAAO,GAAE,cAAmB,GAC1B;IAAE,QAAQ,EAAE,eAAe,EAAE,CAAC;IAAC,gBAAgB,EAAE,eAAe,EAAE,CAAA;CAAE,CAiFtE;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAmBnF"}

package/out/tools/sub-venue/harvest.js

@@ -1,168 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
- *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
- *   measured shape of each designator's identifier half.
- *
- *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
- *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
- *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
- *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- *
- *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
- *   over raw OSM at all:
- *
- *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
- *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
- *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
- *   designator token at all. So names are not harvested wholesale — a name contributes only when it
- *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
- *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
- *
- *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
- *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
- *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
- *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
- *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
- *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
- *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
- */
-import { readLocalTextFile } from "@mailwoman/core/fs/readers";
-import { parseJSONStrict } from "@mailwoman/core/json";
-import { TextSpliterator } from "spliterator";
-import { nameContainsSurfaces } from "#tools/sub-venue/surfaces";
-/**
- * Classify an OSM `ref` into an {@link IdentifierShape} class.
- *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
- * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
- * more than one stand: `16-18` and `0/1`.
- */
-export function classifyIdentifier(ref) {
-    const value = ref.trim();
-    if (/^[0-9]+$/.test(value))
-        return "digit";
-    if (/^[A-Za-z]$/.test(value))
-        return "letter";
-    if (/^[A-Za-z]+[0-9]+$/.test(value))
-        return "letter-digit";
-    if (/^[0-9]+[A-Za-z]+$/.test(value))
-        return "digit-letter";
-    if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value))
-        return "range";
-    return "other";
-}
-/**
- * How many real `ref` values each {@link IdentifierShape} keeps.
- *
- * Eight, not "all" and not one. The field exists so a slice author can see what a class actually CONTAINS — GB's
- * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
- * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
- * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
- */
-const IDENTIFIER_EXAMPLES_PER_SHAPE = 8;
-/**
- * Harvest attested phrases and identifier shapes out of a source's rows.
- *
- * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
- * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
- * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
- * own designator is recorded as `context`.
- *
- * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
- */
-export function extractAttestedPhrases(rows, index, options = {}) {
-    const source = options.source ?? "osm";
-    const region = options.region ?? "";
-    /**
-     * `phrase\0lang\0source` → { count, context }.
-     */
-    const surfaceCounts = new Map();
-    /**
-     * `designatorID\0shape` → { count, examples }.
-     */
-    const shapes = new Map();
-    const note = (phrase, lang, sourceTag, context) => {
-        const key = `${phrase}\0${lang}\0${sourceTag}`;
-        const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map() };
-        entry.count++;
-        entry.context.set(context, (entry.context.get(context) ?? 0) + 1);
-        surfaceCounts.set(key, entry);
-    };
-    for (const row of rows) {
-        if (row.name) {
-            for (const hit of nameContainsSurfaces(row.name, index)) {
-                // `und` — the default `name` tag carries no language. Overture's `name` is the same: a
-                // primary name in whatever language the place uses, untagged.
-                note(hit, "und", `${source}:name`, row.designatorID);
-            }
-        }
-        for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
-            for (const hit of nameContainsSurfaces(localized, index)) {
-                note(hit, lang, `${source}:name:${lang}`, row.designatorID);
-            }
-        }
-        if (row.ref) {
-            const shape = classifyIdentifier(row.ref);
-            const key = `${row.designatorID}\0${shape}`;
-            const entry = shapes.get(key) ?? { count: 0, examples: new Set() };
-            entry.count++;
-            if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
-                entry.examples.add(row.ref.trim());
-            }
-            shapes.set(key, entry);
-        }
-    }
-    const surfaces = [...surfaceCounts].map(([key, entry]) => {
-        const [phrase, lang, sourceTag] = key.split("\0");
-        const record = index.get(phrase);
-        return {
-            phrase,
-            recordID: record.recordID,
-            recordKind: record.recordKind,
-            lang,
-            region,
-            source: sourceTag,
-            curated: false,
-            observations: entry.count,
-            context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
-        };
-    });
-    const identifierShapes = [...shapes].map(([key, entry]) => {
-        const [designatorID, shape] = key.split("\0");
-        return {
-            designatorID,
-            region,
-            shape,
-            observations: entry.count,
-            examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
-        };
-    });
-    return { surfaces, identifierShapes };
-}
-/**
- * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
- * extract is a build output, and one malformed line should not cost the whole lexicon.
- */
-export async function readSubVenueJSONL(path) {
-    const out = [];
-    // `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
-    // (52 MB for Great Britain), and materializing every segment before reading the first is exactly
-    // what the repo lint rule exists to prevent.
-    for (const line of TextSpliterator.from(await readLocalTextFile(path))) {
-        const trimmed = line.trim();
-        if (!trimmed)
-            continue;
-        try {
-            out.push(parseJSONStrict(trimmed));
-        }
-        catch {
-            continue;
-        }
-    }
-    return out;
-}
-//# sourceMappingURL=harvest.js.map

package/out/tools/sub-venue/harvest.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"harvest.js","sourceRoot":"","sources":["../../../lib/tools/sub-venue/harvest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,oBAAoB,EAAqB,MAAM,2BAA2B,CAAA;AA2BnF;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW;IAC7C,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAExB,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAA;IAE1C,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,QAAQ,CAAA;IAE7C,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE1D,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,cAAc,CAAA;IAE1D,IAAI,sCAAsC,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,OAAO,CAAA;IAEtE,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,6BAA6B,GAAG,CAAC,CAAA;AAkBvC;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACrC,IAAkC,EAClC,KAAmB,EACnB,UAA0B,EAAE;IAE5B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IACtC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,EAAE,CAAA;IACnC;;OAEG;IACH,MAAM,aAAa,GAAG,IAAI,GAAG,EAA2D,CAAA;IACxF;;OAEG;IACH,MAAM,MAAM,GAAG,IAAI,GAAG,EAAoD,CAAA;IAE1E,MAAM,IAAI,GAAG,CAAC,MAAc,EAAE,IAAY,EAAE,SAAiB,EAAE,OAAe,EAAQ,EAAE;QACvF,MAAM,GAAG,GAAG,GAAG,MAAM,KAAK,IAAI,KAAK,SAAS,EAAE,CAAA;QAC9C,MAAM,KAAK,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,IAAI,GAAG,EAAkB,EAAE,CAAA;QAExF,KAAK,CAAC,KAAK,EAAE,CAAA;QACb,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACjE,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IAC9B,CAAC,CAAA;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;YACd,KAAK,MAAM,GAAG,IAAI,oBAAoB,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,CAAC;gBACzD,uFAAuF;gBACvF,8DAA8D;gBAC9D,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,GAAG,MAAM,OAAO,EAAE,GAAG,CAAC,YAAY,CAAC,CAAA;YACrD,CAAC;QACF,CAAC;QAED,KAAK,MAAM,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAC,EAAE,CAAC;YAC1E,KAAK,MAAM,GAAG,IAAI,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,CAAC;gBAC1D,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,MAAM,SAAS,IAAI,EAAE,EAAE,GAAG,CAAC,YAAY,CAAC,CAAA;YAC5D,CAAC;QACF,CAAC;QAED,IAAI,GAAG,CAAC,GAAG,EAAE,CAAC;YACb,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACzC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,YAAY,KAAK,KAAK,EAAE,CAAA;YAC3C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,GAAG,EAAU,EAAE,CAAA;YAE1E,KAAK,CAAC,KAAK,EAAE,CAAA;YAEb,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,GAAG,6BAA6B,EAAE,CAAC;gBACzD,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;YACnC,CAAC;YAED,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAsB,CAAC,GAAG,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC3E,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,SAAS,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAA6B,CAAA;QAC7E,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAE,CAAA;QAEjC,OAAO;YACN,MAAM;YACN,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,IAAI;YACJ,MAAM;YACN,MAAM,EAAE,SAAS;YACjB,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,KAAK,CAAC,KAAK;YACzB,OAAO,EAAE,MAAM,CAAC,WAAW,CAAC,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3G,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,gBAAgB,GAAsB,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE;QAC5E,MAAM,CAAC,YAAY,EAAE,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAqB,CAAA;QAEjE,OAAO;YACN,YAAY;YACZ,MAAM;YACN,KAAK;YACL,YAAY,EAAE,KAAK,CAAC,KAAK;YACzB,QAAQ,EAAE,CAAC,GAAG,KAAK,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;SACpE,CAAA;IACF,CAAC,CAAC,CAAA;IAEF,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAA;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,IAAY;IACnD,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,8FAA8F;IAC9F,iGAAiG;IACjG,6CAA6C;IAC7C,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,iBAAiB,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QACxE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,IAAI,CAAC;YACJ,GAAG,CAAC,IAAI,CAAC,eAAe,CAAqB,OAAO,CAAC,CAAC,CAAA;QACvD,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}

package/out/tools/sub-venue/head-nouns.d.ts

@@ -1,50 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file Head-noun derivation for the sub-venue lexicon — proposing the ADDRESSED form of a designator
- *   from the encyclopaedic label a vocabulary source carries.
- *
- *   Wikidata's label for a concept is the encyclopaedic name (`terminal aeroportuaria`, `letištní
- *   terminál`, `havalimanı terminali`), while the addressed form is the bare head (`Terminal`,
- *   `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so the head has to be
- *   extracted before the curation pass has anything to decide about — that gap is why the first wave of
- *   this table shipped 1,014 uncurated surfaces.
- *
- *   Everything derived lands `curated: false`. A derivation is a HYPOTHESIS about what the addressed
- *   form is; a locale's own data is what confirms or kills it.
- */
-import type { SubVenueSurface } from "#tools/sub-venue/table";
-/**
- * Derive the HEAD NOUN of every multi-part surface, so `terminal aeroportuaria` contributes the form anyone actually
- * writes on an envelope.
- *
- * The problem this solves is the whole reason wave 1 shipped 1,014 uncurated surfaces: Wikidata's label for a concept
- * is the ENCYCLOPAEDIC name (`terminal aeroportuaria`, `letištní terminál`, `havalimanı terminali`), while the
- * addressed form is the bare head (`Terminal`, `Terminál`, `Terminali`). Nothing can promote the encyclopaedic form, so
- * the head has to be extracted before the curation pass has anything to decide about.
- *
- * Two derivations, because the table holds two kinds of writing:
- *
- * - **Latin script — the COGNATE test.** A token is the head when its ASCII fold shares {@link HEAD_NOUN_PREFIX_FLOOR}
- *   leading characters with the designator's own canonical id. Nothing subtler survived contact with the data: an
- *   earlier version matched a token against any SINGLE-TOKEN surface of the record, and because Dutch `universiteit` is
- *   a one-token surface of `campus`, it derived `universitario`, `universitaire`, `üniversite` and twenty more as head
- *   nouns of `campus`. Those are the MODIFIER half of the label, and admitting them would have taught the harvest to
- *   read "Ciudad Universitaria" as sub-venue structure.
- * - **Non-Latin script — the SHARED-SUBSTRING test.** The cognate test cannot reach a script the id is not written in,
- *   and for Han and Kana a token split finds nothing at all. So every substring of length ≥
- *   {@link NON_LATIN_HEAD_MIN_LENGTH} occurring in at least two DISTINCT surfaces of the same record and primary
- *   language becomes a candidate, ranked by how many surfaces carry it. Japanese yields `ターミナル` (in all five `ja`
- *   terminal labels) ahead of `ターミナルビル` (three); Chinese yields `航站`, `航站楼`, `航站樓`. Where the script DOES space its
- *   words (Korean, Greek, Cyrillic) a candidate must be a whole token, so `공항 터미널` ∩ `공항터미널` gives `터미널` and never a
- *   fragment.
- *
- * The non-Latin branch deliberately emits SEVERAL candidates instead of picking one. Choosing between `航站` and `航站楼`
- * from Wikidata alone is guesswork; the Japan extract answers it by counting, and the promotion ledger records which
- * count won. Everything derived lands `curated: false` — the derivation is a hypothesis about what the addressed form
- * is, and a locale's own data is what confirms or kills it.
- */
-export declare function deriveHeadNounSurfaces(surfaces: readonly SubVenueSurface[]): SubVenueSurface[];
-//# sourceMappingURL=head-nouns.d.ts.map

package/out/tools/sub-venue/head-nouns.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"head-nouns.d.ts","sourceRoot":"","sources":["../../../lib/tools/sub-venue/head-nouns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAIH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAuD7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,wBAAgB,sBAAsB,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,GAAG,eAAe,EAAE,CA+E9F"}

package/out/tools/sub-venue/head-nouns.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"head-nouns.js","sourceRoot":"","sources":["../../../lib/tools/sub-venue/head-nouns.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AAInD;;;;;GAKG;AACH,SAAS,SAAS,CAAC,IAAY;IAC9B,OAAO,IAAI;SACT,SAAS,CAAC,KAAK,CAAC;SAChB,UAAU,CAAC,iBAAiB,EAAE,EAAE,CAAC;SACjC,WAAW,EAAE,CAAA;AAChB,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAEhC;;;GAGG;AACH,MAAM,yBAAyB,GAAG,CAAC,CAAA;AAEnC;;;;GAIG;AACH,MAAM,4BAA4B,GAAG,CAAC,CAAA;AAEtC;;GAEG;AACH,MAAM,YAAY,GAAG,iCAAiC,CAAA;AAEtD;;;;;;;;;GASG;AACH,MAAM,uBAAuB,GAAG,0EAA0E,CAAA;AAE1G;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,MAAM,UAAU,sBAAsB,CAAC,QAAoC;IAC1E,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAClD,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IAElF,MAAM,IAAI,GAAG,CAAC,MAAc,EAAE,IAAqB,EAAQ,EAAE;QAC5D,IAAI,MAAM,KAAK,IAAI,CAAC,MAAM;YAAE,OAAM;QAElC,MAAM,GAAG,GAAG,GAAG,MAAM,KAAK,IAAI,CAAC,QAAQ,KAAK,IAAI,CAAC,IAAI,EAAE,CAAA;QAEvD,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAM;QAE7C,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE;YAChB,MAAM;YACN,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,mBAAmB;YAC3B,OAAO,EAAE,KAAK;YACd,YAAY,EAAE,CAAC;YACf,OAAO,EAAE,EAAE;SACX,CAAC,CAAA;IACH,CAAC,CAAA;IAED,oGAAoG;IACpG,gFAAgF;IAChF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAEhD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;QAEvE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,SAAQ;QAE9B,MAAM,IAAI,GAAG,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAE3D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YAC1B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;YAE9B,IAAI,MAAM,CAAC,MAAM,IAAI,KAAK,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;gBACzE,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACpB,CAAC;QACF,CAAC;IACF,CAAC;IAED,6FAA6F;IAC7F,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAA;IAE7C,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE3D,gGAAgG;QAChG,gFAAgF;QAChF,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,EAAE,CAAA;QACpE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,GAAG,EAAU,CAAA;QACjD,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACxB,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IACtB,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAoB,CAAA;IAErD,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,SAAQ;QAC3B,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,yBAAyB,CAAC,IAAI,CAAC,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC,uBAAuB,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE3D,MAAM,GAAG,GAAG,GAAG,OAAO,CAAC,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAE,EAAE,CAAA;QAEpE,KAAK,MAAM,SAAS,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;YAC1D,IAAI,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;gBACxC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;YACzB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;AAC7B,CAAC;AAED;;GAEG;AACH,SAAS,kBAAkB,CAAC,CAAS,EAAE,CAAS;IAC/C,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAA;IAC1C,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,CAAC,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACnC,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,yBAAyB,CAAC,IAAyB;IAC3D,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACzB,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC3D,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC3G,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAA;IAExC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAA;QAE/B,KAAK,IAAI,MAAM,GAAG,yBAAyB,EAAE,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,CAAC;YAChF,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,IAAI,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;gBAC9D,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAA;gBAErD,IAAI,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC;oBAAE,SAAQ;gBACnC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;YAC/B,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACxD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAA;IAEzG,OAAO,IAAI;SACT,MAAM,CAAC,CAAC,CAAC,SAAS,EAAE,KAAK,CAAC,EAAE,EAAE,CAC9B,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,EAAE,CAAC,KAAK,KAAK,SAAS,IAAI,UAAU,KAAK,KAAK,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAC9G;SACA,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;SACxF,KAAK,CAAC,CAAC,EAAE,4BAA4B,CAAC;SACtC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,SAAS,CAAC,CAAA;AAClC,CAAC"}

package/out/tools/sub-venue/surfaces.d.ts

@@ -1,45 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file Surface-form primitives for the sub-venue lexicon: the normalizer every phrase passes
- *   through, the phrase → record index built over the surfaces accumulated so far, and the operator
- *   that decides whether a feature's name contains a known designator.
- *
- *   The two matching rules here are script-conditional, and both narrowings are required — see
- *   {@link NON_SPACING_SCRIPT} and {@link nameContainsSurfaces}.
- */
-import type { SubVenueSurface } from "#tools/sub-venue/table";
-/**
- * Normalize a surface for the table: trim, collapse internal whitespace, and lowercase ONLY when the string is entirely
- * in a bicameral script. `ターミナルビル` and `航站楼` pass through untouched; `Flughafenterminal` folds.
- */
-export declare function normalizeSurface(text: string): string;
-/**
- * A phrase → record index, keyed on the normalized phrase. Built by {@link buildSurfaceIndex} from the surfaces present
- * before the harvest runs, and the reason a matched phrase can be attributed to the record it actually names.
- */
-export type SurfaceIndex = ReadonlyMap<string, {
-    recordID: string;
-    recordKind: "designator" | "modifier";
-}>;
-/**
- * Index the surfaces accumulated so far by phrase. First writer wins, so a seed record beats a Wikidata alias that
- * happens to collide — `terminal` stays the `terminal` designator even though it is also an Italian alias for it.
- */
-export declare function buildSurfaceIndex(surfaces: readonly SubVenueSurface[]): SurfaceIndex;
-/**
- * Every known phrase found in `name`, as whole-token runs for spacing scripts and as substrings for non-spacing ones.
- *
- * Token-boundary matching for Latin script, not substring: `Nordterminal` is a real German compound in which `terminal`
- * is a suffix, and a substring test would also fire on `Terminalstraße`. The compound case is a genuine miss and it is
- * the right miss — admitting suffix matches would fire on every `-hall`/`-gate` compound in Germanic and Nordic street
- * naming, which is exactly the confound class `Briggate`/`Kirkgate` represents.
- *
- * For Han/Kana names that rule finds nothing at all, because the script has no word boundaries: `第1ターミナル` splits into
- * one token that matches no surface. There the LONGEST known substring is the correct operator, and the compound
- * objection does not transfer — Japanese has no `-gate` street-name suffix class.
- */
-export declare function nameContainsSurfaces(name: string, index: SurfaceIndex): string[];
-//# sourceMappingURL=surfaces.d.ts.map

package/out/tools/sub-venue/surfaces.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"surfaces.d.ts","sourceRoot":"","sources":["../../../lib/tools/sub-venue/surfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAA;AAiB7D;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAIrD;AAED;;;GAGG;AACH,MAAM,MAAM,YAAY,GAAG,WAAW,CAAC,MAAM,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,YAAY,GAAG,UAAU,CAAA;CAAE,CAAC,CAAA;AAE3G;;;GAGG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,SAAS,eAAe,EAAE,GAAG,YAAY,CASpF;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,oBAAoB,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,GAAG,MAAM,EAAE,CAqBhF"}

package/out/tools/sub-venue/surfaces.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"surfaces.js","sourceRoot":"","sources":["../../../lib/tools/sub-venue/surfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAIH;;GAEG;AACH,MAAM,mBAAmB,GAAG,oEAAoE,CAAA;AAEhG;;;;;;;GAOG;AACH,MAAM,kBAAkB,GAAG,yDAAyD,CAAA;AAEpF;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAY;IAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;IAEpD,OAAO,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAA;AAC3E,CAAC;AAQD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAoC;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,EAAuE,CAAA;IAE5F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QACvC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,CAAC,UAAU,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,KAAmB;IACrE,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IACzC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,KAAK,IAAI,UAAU,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC;QACpD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,mCAAmC,EAAE,EAAE,CAAC,CAAA;QAE1E,IAAI,QAAQ,IAAI,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;QACnB,CAAC;IACF,CAAC;IAED,IAAI,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QACzC,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YAC9B,IAAI,kBAAkB,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YACjB,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,CAAC,GAAG,IAAI,CAAC,CAAA;AACjB,CAAC"}