@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/tools/lint-slice-vocab.ts

@@ -1,388 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
- *
- *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
- *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
- */
-
-import { dataRootPath } from "@mailwoman/core/data-root"
-import { readDirectory } from "@mailwoman/core/fs/readers"
-import { tryParsingJSON } from "@mailwoman/core/json"
-import { pyRound } from "@mailwoman/core/numeric"
-import { join } from "path-ts"
-
-import { connectDuckDB, type DuckDBConnection } from "#utils/parquet"
-
-/**
- * A column-projected base/slice row: parallel token + label lists plus the row's country.
- */
-interface CorpusRow {
-	tokens: string[]
-	labels: string[]
-	country: string | null
-}
-
-/**
- * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
- */
-function stripBIO(label: string): string {
-	const head = label.slice(0, 2)
-
-	return head === "B-" || head === "I-" ? label.slice(2) : label
-}
-
-/**
- * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
- * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
- * corpora actually contain.
- */
-function isDigit(token: string): boolean {
-	return token.length > 0 && /^\p{Nd}+$/u.test(token)
-}
-
-/**
- * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
- */
-function pct(frac: number): string {
-	return `${pyRound(frac * 100)}%`
-}
-
-/**
- * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
- * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
- * print.
- */
-function formatPyFloat(n: number): string {
-	return Number.isInteger(n) ? n.toFixed(1) : String(n)
-}
-
-/**
- * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
- */
-function pad(value: string, width: number): string {
-	return value.padEnd(width)
-}
-
-/**
- * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
- * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
- */
-function dominant(counter: Map<string, number>): [string, number, number] {
-	let total = 0
-	let bestTag = ""
-	let bestCount = -1
-
-	for (const [tag, count] of counter) {
-		total += count
-
-		if (count > bestCount) {
-			bestCount = count
-			bestTag = tag
-		}
-	}
-
-	if (total === 0) return ["", 0, 0]
-
-	return [bestTag, total, bestCount / total]
-}
-
-/**
- * Bump a (key -> count) tally, creating the inner counter on first sight.
- */
-function bump(table: Map<string, Map<string, number>>, key: string, sub: string): void {
-	let counter = table.get(key)
-
-	if (!counter) {
-		counter = new Map()
-		table.set(key, counter)
-	}
-
-	counter.set(sub, (counter.get(sub) ?? 0) + 1)
-}
-
-/**
- * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
- * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
- * here.
- */
-async function readRows(con: DuckDBConnection, path: string): Promise<CorpusRow[]> {
-	const result = await con.runAndReadAll(
-		`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`
-	)
-
-	const raw = result.getRowObjects() as Array<{ tokens: unknown; labels: unknown; country: unknown }>
-	const rows: CorpusRow[] = []
-
-	for (const r of raw) {
-		const tokens = tryParsingJSON(String(r.tokens))
-		const labels = tryParsingJSON(String(r.labels))
-
-		if (!Array.isArray(tokens) || !Array.isArray(labels)) continue
-
-		rows.push({
-			tokens: tokens as string[],
-			labels: labels as string[],
-			country: r.country == null ? null : String(r.country),
-		})
-	}
-
-	return rows
-}
-
-/**
- * Read just the first row's `source` value — used to group base parts for a proportional slice.
- */
-async function readSource(con: DuckDBConnection, path: string): Promise<string> {
-	const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`)
-	const rows = result.getRowObjects() as Array<{ source: unknown }>
-
-	return rows.length ? String(rows[0]!.source) : ""
-}
-
-/**
- * Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`.
- */
-async function globParquet(dir: string): Promise<string[]> {
-	let names: string[]
-
-	try {
-		names = await readDirectory(dir)
-	} catch {
-		return []
-	}
-
-	return names
-		.filter((f) => f.endsWith(".parquet"))
-		.map((f) => join(dir, f))
-		.toSorted()
-}
-
-/**
- * Options for {@linkcode lintSliceVocab}.
- */
-export interface LintSliceVocabOptions {
-	/**
-	 * The slice parquet to lint.
-	 */
-	slice: string
-	/**
-	 * Base corpus version. Default `v0.5.0`.
-	 */
-	baseVersion?: string
-	/**
-	 * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
-	 */
-	baseRoot?: string
-	/**
-	 * Base-majority confidence floor for a contradiction. Default 0.7.
-	 */
-	threshold?: number
-	/**
-	 * Minimum base support to judge a token. Default 50.
-	 */
-	minCount?: number
-	/**
-	 * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
-	 */
-	fraction?: number
-}
-
-/**
- * One contradiction row: token, slice tag, base tag, base fraction, base total.
- */
-export type SliceVocabRow = [token: string, sliceTag: string, baseTag: string, baseFrac: number, baseTotal: number]
-
-/**
- * Findings summary returned by {@linkcode lintSliceVocab}.
- */
-export interface LintSliceVocabSummary {
-	/**
-	 * Real contradictions — the command exits 1 when nonzero.
-	 */
-	errors: number
-	/**
-	 * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
-	 */
-	warnings: number
-	findings: { contradictions: SliceVocabRow[]; affixSplits: SliceVocabRow[] }
-}
-
-/**
- * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
- */
-export async function lintSliceVocab(options: LintSliceVocabOptions): Promise<LintSliceVocabSummary> {
-	const baseVersion = options.baseVersion ?? "v0.5.0"
-	const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned")
-	const threshold = options.threshold ?? 0.7
-	const minCount = options.minCount ?? 50
-	const fraction = options.fraction ?? 1
-
-	const con = await connectDuckDB()
-
-	// 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
-	const sliceRows = await readRows(con, options.slice)
-	const sliceTags = new Map<string, Map<string, number>>()
-	const sliceCountries = new Map<string, Set<string | null>>()
-
-	for (const { tokens, labels, country } of sliceRows) {
-		const n = Math.min(tokens.length, labels.length)
-
-		for (let i = 0; i < n; i++) {
-			const w = tokens[i]!
-			const l = labels[i]!
-
-			if (isDigit(w)) continue // numbers are context-determined (house_number/postcode), not lexical vocab
-			bump(sliceTags, w, stripBIO(l))
-			let set = sliceCountries.get(w)
-
-			if (!set) {
-				set = new Set()
-				sliceCountries.set(w, set)
-			}
-
-			set.add(country)
-		}
-	}
-
-	const sliceVocab = new Set(sliceTags.keys())
-
-	console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`)
-
-	// 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
-	const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train")
-	let parts = await globParquet(trainDir)
-
-	if (!parts.length) {
-		throw new Error("no base parts found")
-	}
-
-	if (fraction < 1) {
-		const bysrc = new Map<string, string[]>()
-
-		for (const p of parts) {
-			const src = await readSource(con, p)
-			let list = bysrc.get(src)
-
-			if (!list) {
-				list = []
-				bysrc.set(src, list)
-			}
-
-			list.push(p)
-		}
-
-		const sliced: string[] = []
-
-		for (const ps of bysrc.values()) {
-			const take = Math.max(2, pyRound(ps.length * fraction))
-
-			for (const p of ps.slice(0, take)) {
-				sliced.push(p)
-			}
-		}
-
-		parts = sliced
-	}
-
-	console.log(
-		`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`
-	)
-
-	// 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
-	const baseTags = new Map<string, Map<string, number>>()
-
-	for (let i = 0; i < parts.length; i++) {
-		const rows = await readRows(con, parts[i]!)
-
-		for (const { tokens, labels, country } of rows) {
-			const n = Math.min(tokens.length, labels.length)
-
-			for (let j = 0; j < n; j++) {
-				const w = tokens[j]!
-
-				if (sliceVocab.has(w) && sliceCountries.get(w)!.has(country)) {
-					bump(baseTags, w, stripBIO(labels[j]!))
-				}
-			}
-		}
-
-		if ((i + 1) % 100 === 0) {
-			console.log(`  ...${i + 1}/${parts.length} parts`)
-		}
-	}
-
-	// 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
-	const flagged: SliceVocabRow[] = []
-	const affix: SliceVocabRow[] = []
-
-	for (const w of sliceVocab) {
-		const [sTag] = dominant(sliceTags.get(w)!)
-		const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map())
-
-		if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold) continue
-		const row: SliceVocabRow = [w, sTag, bTag, bFrac, bTotal]
-
-		if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
-			affix.push(row)
-		} else {
-			flagged.push(row)
-		}
-	}
-
-	const sections: Array<[string, SliceVocabRow[]]> = [
-		["CONTRADICTION", flagged],
-		["affix-split (EXPECTED — affix-relabel handles)", affix],
-	]
-
-	for (const [label, rows] of sections) {
-		if (!rows.length) continue
-		rows.sort((a, b) => b[4] - a[4] || b[3] - a[3])
-
-		console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`)
-
-		for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
-			console.log(`  ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`)
-		}
-	}
-
-	if (!flagged.length) {
-		console.log(
-			`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`
-		)
-	}
-
-	return {
-		errors: flagged.length,
-		warnings: affix.length,
-		findings: { contradictions: flagged, affixSplits: affix },
-	}
-}

package/lib/tools/sub-venue/harvest.ts

@@ -1,234 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   @file The harvest side of the sub-venue lexicon: what one row of a harvestable source looks like,
- *   how a file of them is read back, and what a pass over them extracts — attested surfaces plus the
- *   measured shape of each designator's identifier half.
- *
- *   {@link SubVenueHarvestRow} is SOURCE-NEUTRAL, and verified so: an Overture Places row from the
- *   `airport_terminal` category is `{ designatorID, name }` and fits unchanged. It is declared here
- *   rather than imported so the builder does not take a dependency on `@mailwoman/osm` (which
- *   `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- *
- *   Two measurements shape {@link extractAttestedPhrases}, and both are the reason it is safe to run
- *   over raw OSM at all:
- *
- *   **A feature's `name` is usually the VENUE's name, not a sub-venue phrase.** On the Berlin extract
- *   (2,060 matched features) a `railway=platform` is named `Stendaler Straße` and an `amenity=university`
- *   is named `Hertie School`; across 250,116 named Great Britain features only 6,003 (2.40%) contain a
- *   designator token at all. So names are not harvested wholesale — a name contributes only when it
- *   CONTAINS a phrase already in the surface index, which is what makes `Terminal E (Untere Ebene)`
- *   evidence and `Otto Lilienthal Flughafen Berlin Tegel` not.
- *
- *   **A matched phrase belongs to the record the PHRASE names, not the record the ROW carries.**
- *   Attributing every hit to `row.designatorID` — the rule that matched the FEATURE — produced `west →
- *   platform`, `hall → platform` and `biggin → platform` on the GB extract, because a bus stop tagged
- *   `public_transport=platform` is named "Village Hall" or "West Kensington"; 108 of 133 OSM-derived
- *   surfaces named a different record than the one they pointed at. Attribution therefore runs through
- *   the index, and the row's own designator is kept as `context` — exactly the axis a confound board
- *   needs, since a `hall` seen on a platform is a confound and a `hall` seen on a terminal is evidence.
- */
-
-import { readLocalTextFile } from "@mailwoman/core/fs/readers"
-import { parseJSONStrict } from "@mailwoman/core/json"
-import { TextSpliterator } from "spliterator"
-
-import { nameContainsSurfaces, type SurfaceIndex } from "#tools/sub-venue/surfaces"
-import type { IdentifierShape, SubVenueSurface } from "#tools/sub-venue/table"
-
-/**
- * One row of a harvestable source, as read back off JSONL or out of a layer database.
- *
- * SOURCE-NEUTRAL by design, and verified so in wave 2: an Overture Places row from the `airport_terminal` category is
- * `{ designatorID, name }` and fits unchanged. What did NOT fit was the harvest function's hardcoded `osm:name` source
- * stamp — see `overture-subvenue.ts`'s docstring. Declared locally so the builder does not import `@mailwoman/osm`
- * (which `@mailwoman/corpus` does not depend on) just to name a shape it reads from a file.
- */
-export interface SubVenueHarvestRow {
-	/**
-	 * The designator the SOURCE's rule assigned to the FEATURE. Not necessarily the record a matched phrase names — see
-	 * this file's header on phrase attribution. Carried into {@link SubVenueSurface.context}.
-	 */
-	designatorID: string
-	/**
-	 * `venue` (station, airport, campus) or `sub_venue`, when the source's rule assigns one. The harvest itself does not
-	 * read it; the sub-venue slice recipe fills its venue slot from it.
-	 */
-	tier?: string
-	name?: string | null
-	ref?: string | null
-	localizedNames?: Record<string, string>
-}
-
-/**
- * Classify an OSM `ref` into an {@link IdentifierShape} class.
- *
- * The classes are the ones Berlin's gates actually produced, plus the two aviation forms the corpus task names
- * (`Terminal 2F` is digit-letter, `Concourse B` is letter). `range` covers both separators OSM uses for a gate serving
- * more than one stand: `16-18` and `0/1`.
- */
-export function classifyIdentifier(ref: string): string {
-	const value = ref.trim()
-
-	if (/^[0-9]+$/.test(value)) return "digit"
-
-	if (/^[A-Za-z]$/.test(value)) return "letter"
-
-	if (/^[A-Za-z]+[0-9]+$/.test(value)) return "letter-digit"
-
-	if (/^[0-9]+[A-Za-z]+$/.test(value)) return "digit-letter"
-
-	if (/^[0-9A-Za-z]+\s*[-/]\s*[0-9A-Za-z]+$/.test(value)) return "range"
-
-	return "other"
-}
-
-/**
- * How many real `ref` values each {@link IdentifierShape} keeps.
- *
- * Eight, not "all" and not one. The field exists so a slice author can see what a class actually CONTAINS — GB's
- * `other` class turned out to be semicolon multi-values (`1;2;3`, `13;14`), which one example would have hidden and
- * which the class name does not say. Eight fits a terminal line and covers the variety inside every class the GB
- * extract produced. The COUNT lives in `observations`; this is a sample, not a census.
- */
-const IDENTIFIER_EXAMPLES_PER_SHAPE = 8
-
-/**
- * Options for one harvest pass — which source stamp its surfaces carry and which region they were attested in.
- *
- * Both default to the OSM/unknown-region values wave 1 hardcoded, so an existing caller is unchanged.
- */
-export interface HarvestOptions {
-	/**
-	 * Source family: `osm` yields `osm:name` / `osm:name:<lang>`, `overture` yields `overture:name`.
-	 */
-	source?: string
-	/**
-	 * ISO 3166-1 alpha-2 of the extract or partition. `""` when unknown.
-	 */
-	region?: string
-}
-
-/**
- * Harvest attested phrases and identifier shapes out of a source's rows.
- *
- * `index` gates the name harvest — a name contributes only when it CONTAINS a phrase already in the table. That filter
- * is the whole reason this function is safe to run over raw OSM: see this file's header for the Berlin measurement that
- * motivated it. The index also decides ATTRIBUTION: a hit is a surface of the record the PHRASE names, and the row's
- * own designator is recorded as `context`.
- *
- * Returns surfaces with real `observations` counts, so the lexicon can rank `terminal` above a phrase attested once.
- */
-export function extractAttestedPhrases(
-	rows: Iterable<SubVenueHarvestRow>,
-	index: SurfaceIndex,
-	options: HarvestOptions = {}
-): { surfaces: SubVenueSurface[]; identifierShapes: IdentifierShape[] } {
-	const source = options.source ?? "osm"
-	const region = options.region ?? ""
-	/**
-	 * `phrase\0lang\0source` → { count, context }.
-	 */
-	const surfaceCounts = new Map<string, { count: number; context: Map<string, number> }>()
-	/**
-	 * `designatorID\0shape` → { count, examples }.
-	 */
-	const shapes = new Map<string, { count: number; examples: Set<string> }>()
-
-	const note = (phrase: string, lang: string, sourceTag: string, context: string): void => {
-		const key = `${phrase}\0${lang}\0${sourceTag}`
-		const entry = surfaceCounts.get(key) ?? { count: 0, context: new Map<string, number>() }
-
-		entry.count++
-		entry.context.set(context, (entry.context.get(context) ?? 0) + 1)
-		surfaceCounts.set(key, entry)
-	}
-
-	for (const row of rows) {
-		if (row.name) {
-			for (const hit of nameContainsSurfaces(row.name, index)) {
-				// `und` — the default `name` tag carries no language. Overture's `name` is the same: a
-				// primary name in whatever language the place uses, untagged.
-				note(hit, "und", `${source}:name`, row.designatorID)
-			}
-		}
-
-		for (const [lang, localized] of Object.entries(row.localizedNames ?? {})) {
-			for (const hit of nameContainsSurfaces(localized, index)) {
-				note(hit, lang, `${source}:name:${lang}`, row.designatorID)
-			}
-		}
-
-		if (row.ref) {
-			const shape = classifyIdentifier(row.ref)
-			const key = `${row.designatorID}\0${shape}`
-			const entry = shapes.get(key) ?? { count: 0, examples: new Set<string>() }
-
-			entry.count++
-
-			if (entry.examples.size < IDENTIFIER_EXAMPLES_PER_SHAPE) {
-				entry.examples.add(row.ref.trim())
-			}
-
-			shapes.set(key, entry)
-		}
-	}
-
-	const surfaces: SubVenueSurface[] = [...surfaceCounts].map(([key, entry]) => {
-		const [phrase, lang, sourceTag] = key.split("\0") as [string, string, string]
-		const record = index.get(phrase)!
-
-		return {
-			phrase,
-			recordID: record.recordID,
-			recordKind: record.recordKind,
-			lang,
-			region,
-			source: sourceTag,
-			curated: false,
-			observations: entry.count,
-			context: Object.fromEntries([...entry.context].toSorted((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))),
-		}
-	})
-
-	const identifierShapes: IdentifierShape[] = [...shapes].map(([key, entry]) => {
-		const [designatorID, shape] = key.split("\0") as [string, string]
-
-		return {
-			designatorID,
-			region,
-			shape,
-			observations: entry.count,
-			examples: [...entry.examples].toSorted((a, b) => a.localeCompare(b)),
-		}
-	})
-
-	return { surfaces, identifierShapes }
-}
-
-/**
- * Read a JSONL file of {@link SubVenueHarvestRow}s. Blank lines and unparseable rows are skipped rather than fatal — an
- * extract is a build output, and one malformed line should not cost the whole lexicon.
- */
-export async function readSubVenueJSONL(path: string): Promise<SubVenueHarvestRow[]> {
-	const out: SubVenueHarvestRow[] = []
-
-	// `TextSpliterator` rather than `split("\n")` — a whole-country extract runs to 250,000 lines
-	// (52 MB for Great Britain), and materializing every segment before reading the first is exactly
-	// what the repo lint rule exists to prevent.
-	for (const line of TextSpliterator.from(await readLocalTextFile(path))) {
-		const trimmed = line.trim()
-
-		if (!trimmed) continue
-
-		try {
-			out.push(parseJSONStrict<SubVenueHarvestRow>(trimmed))
-		} catch {
-			continue
-		}
-	}
-
-	return out
-}