@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/tools/lint/slice/vocab.ts

@@ -0,0 +1,376 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
+ *
+ *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
+ *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { tryParsingJSON } from "@mailwoman/core/json"
+import { pyRound } from "@mailwoman/core/numeric"
+import { join } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
+
+import { connectDuckDB, type DuckDBConnection } from "#utils/parquet"
+
+/**
+ * A column-projected base/slice row: parallel token + label lists plus the row's country.
+ */
+interface CorpusRow {
+	tokens: string[]
+	labels: string[]
+	country: string | null
+}
+
+/**
+ * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
+ */
+function stripBIO(label: string): string {
+	const head = label.slice(0, 2)
+
+	return head === "B-" || head === "I-" ? label.slice(2) : label
+}
+
+/**
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
+ * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
+ * corpora actually contain.
+ */
+function isDigit(token: string): boolean {
+	return token.length > 0 && /^\p{Nd}+$/u.test(token)
+}
+
+/**
+ * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
+ */
+function pct(frac: number): string {
+	return `${pyRound(frac * 100)}%`
+}
+
+/**
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
+ * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
+ * print.
+ */
+function formatPyFloat(n: number): string {
+	return Number.isInteger(n) ? n.toFixed(1) : String(n)
+}
+
+/**
+ * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
+ */
+function pad(value: string, width: number): string {
+	return value.padEnd(width)
+}
+
+/**
+ * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
+ * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
+ */
+function dominant(counter: Map<string, number>): [string, number, number] {
+	let total = 0
+	let bestTag = ""
+	let bestCount = -1
+
+	for (const [tag, count] of counter) {
+		total += count
+
+		if (count > bestCount) {
+			bestCount = count
+			bestTag = tag
+		}
+	}
+
+	if (total === 0) return ["", 0, 0]
+
+	return [bestTag, total, bestCount / total]
+}
+
+/**
+ * Bump a (key -> count) tally, creating the inner counter on first sight.
+ */
+function bump(table: Map<string, Map<string, number>>, key: string, sub: string): void {
+	let counter = table.get(key)
+
+	if (!counter) {
+		counter = new Map()
+		table.set(key, counter)
+	}
+
+	counter.set(sub, (counter.get(sub) ?? 0) + 1)
+}
+
+/**
+ * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
+ * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
+ * here.
+ */
+async function readRows(con: DuckDBConnection, path: string): Promise<CorpusRow[]> {
+	const result = await con.runAndReadAll(
+		`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`
+	)
+
+	const raw = result.getRowObjects() as Array<{ tokens: unknown; labels: unknown; country: unknown }>
+	const rows: CorpusRow[] = []
+
+	for (const r of raw) {
+		const tokens = tryParsingJSON(String(r.tokens))
+		const labels = tryParsingJSON(String(r.labels))
+
+		if (!Array.isArray(tokens) || !Array.isArray(labels)) continue
+
+		rows.push({
+			tokens: tokens as string[],
+			labels: labels as string[],
+			country: r.country == null ? null : String(r.country),
+		})
+	}
+
+	return rows
+}
+
+/**
+ * Read just the first row's `source` value — used to group base parts for a proportional slice.
+ */
+async function readSource(con: DuckDBConnection, path: string): Promise<string> {
+	const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`)
+	const rows = result.getRowObjects() as Array<{ source: unknown }>
+
+	return rows.length ? String(rows[0]!.source) : ""
+}
+
+/**
+ * Options for {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabOptions {
+	/**
+	 * The slice parquet to lint.
+	 */
+	slice: string
+	/**
+	 * Base corpus version. Default `v0.5.0`.
+	 */
+	baseVersion?: string
+	/**
+	 * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
+	 */
+	baseRoot?: string
+	/**
+	 * Base-majority confidence floor for a contradiction. Default 0.7.
+	 */
+	threshold?: number
+	/**
+	 * Minimum base support to judge a token. Default 50.
+	 */
+	minCount?: number
+	/**
+	 * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
+	 */
+	fraction?: number
+}
+
+/**
+ * One contradiction row: token, slice tag, base tag, base fraction, base total.
+ */
+export type SliceVocabRow = [token: string, sliceTag: string, baseTag: string, baseFrac: number, baseTotal: number]
+
+/**
+ * Findings summary returned by {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabSummary {
+	/**
+	 * Real contradictions — the command exits 1 when nonzero.
+	 */
+	errors: number
+	/**
+	 * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
+	 */
+	warnings: number
+	findings: { contradictions: SliceVocabRow[]; affixSplits: SliceVocabRow[] }
+}
+
+/**
+ * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export async function lintSliceVocab(options: LintSliceVocabOptions): Promise<LintSliceVocabSummary> {
+	const baseVersion = options.baseVersion ?? "v0.5.0"
+	const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned")
+	const threshold = options.threshold ?? 0.7
+	const minCount = options.minCount ?? 50
+	const fraction = options.fraction ?? 1
+
+	const con = await connectDuckDB()
+
+	// 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
+	const sliceRows = await readRows(con, options.slice)
+	const sliceTags = new Map<string, Map<string, number>>()
+	const sliceCountries = new Map<string, Set<string | null>>()
+
+	for (const { tokens, labels, country } of sliceRows) {
+		const n = Math.min(tokens.length, labels.length)
+
+		for (let i = 0; i < n; i++) {
+			const w = tokens[i]!
+			const l = labels[i]!
+
+			if (isDigit(w)) continue // numbers are context-determined (house_number/postcode), not lexical vocab
+			bump(sliceTags, w, stripBIO(l))
+			let set = sliceCountries.get(w)
+
+			if (!set) {
+				set = new Set()
+				sliceCountries.set(w, set)
+			}
+
+			set.add(country)
+		}
+	}
+
+	const sliceVocab = new Set(sliceTags.keys())
+
+	console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`)
+
+	// 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
+	const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train")
+
+	let parts = (
+		await Globerator.from("*.parquet", {
+			cwd: trainDir,
+			absolute: true,
+		}).toArray()
+	).toSorted()
+
+	if (!parts.length) {
+		throw new Error("no base parts found")
+	}
+
+	if (fraction < 1) {
+		const bysrc = new Map<string, string[]>()
+
+		for (const p of parts) {
+			const src = await readSource(con, p)
+			let list = bysrc.get(src)
+
+			if (!list) {
+				list = []
+				bysrc.set(src, list)
+			}
+
+			list.push(p)
+		}
+
+		const sliced: string[] = []
+
+		for (const ps of bysrc.values()) {
+			const take = Math.max(2, pyRound(ps.length * fraction))
+
+			for (const p of ps.slice(0, take)) {
+				sliced.push(p)
+			}
+		}
+
+		parts = sliced
+	}
+
+	console.log(
+		`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`
+	)
+
+	// 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
+	const baseTags = new Map<string, Map<string, number>>()
+
+	for (let i = 0; i < parts.length; i++) {
+		const rows = await readRows(con, parts[i]!)
+
+		for (const { tokens, labels, country } of rows) {
+			const n = Math.min(tokens.length, labels.length)
+
+			for (let j = 0; j < n; j++) {
+				const w = tokens[j]!
+
+				if (sliceVocab.has(w) && sliceCountries.get(w)!.has(country)) {
+					bump(baseTags, w, stripBIO(labels[j]!))
+				}
+			}
+		}
+
+		if ((i + 1) % 100 === 0) {
+			console.log(`  ...${i + 1}/${parts.length} parts`)
+		}
+	}
+
+	// 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
+	const flagged: SliceVocabRow[] = []
+	const affix: SliceVocabRow[] = []
+
+	for (const w of sliceVocab) {
+		const [sTag] = dominant(sliceTags.get(w)!)
+		const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map())
+
+		if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold) continue
+		const row: SliceVocabRow = [w, sTag, bTag, bFrac, bTotal]
+
+		if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
+			affix.push(row)
+		} else {
+			flagged.push(row)
+		}
+	}
+
+	const sections: Array<[string, SliceVocabRow[]]> = [
+		["CONTRADICTION", flagged],
+		["affix-split (EXPECTED — affix-relabel handles)", affix],
+	]
+
+	for (const [label, rows] of sections) {
+		if (!rows.length) continue
+		rows.sort((a, b) => b[4] - a[4] || b[3] - a[3])
+
+		console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`)
+
+		for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
+			console.log(`  ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`)
+		}
+	}
+
+	if (!flagged.length) {
+		console.log(
+			`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`
+		)
+	}
+
+	return {
+		errors: flagged.length,
+		warnings: affix.length,
+		findings: { contradictions: flagged, affixSplits: affix },
+	}
+}

package/lib/tools/overlay-manifest.ts

@@ -4,10 +4,12 @@
  * @author Teffen Ellis, et al.
  *
  *   Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
- *   ADDS one slice parquet to a base corpus, keeping every base slice VERBATIM (pure overlay ADD),
- *   and re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet +
- *   --source so it works for any overlay slice (the fr-admin-split one is the original; #148's
- *   overture-multilocale is the second user).
+ *   ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and
+ *   re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one
+ *   label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's
+ *   overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a
+ *   set rather than one — chaining eight single-slice overlays would leave seven dead directories and
+ *   an eight-deep base chain for what is one version).
  *
  *   Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
  *   read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
@@ -43,14 +45,62 @@ interface SliceDescriptor {
 	source: string
 }
 
+type ManifestSlice = Record<string, unknown> & { path: string; source?: string }
+
+/**
+ * The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every
+ * corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact
+ * — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the
+ * word is banned in this tree and the ratchet's baseline is zero.
+ */
+const LEGACY_SLICES_KEY = `sh${"ards"}` as const
+const LEGACY_ROWS_PER_SLICE_KEY = `rows_per_sh${"ard"}` as const
+
 interface BaseManifest {
 	corpus_version?: string
 	schema: unknown
-	rows_per_slice: unknown
+	rows_per_slice?: unknown
 	row_group_size: unknown
-	slices: Array<Record<string, unknown> & { path: string; source?: string }>
+	slices?: ManifestSlice[]
 	counts: { train: number; val: number; test: number }
 	total_rows: number
+	/**
+	 * The pre-rename spellings, present on every corpus built before 2026-09-01.
+	 */
+	[LEGACY_SLICES_KEY]?: ManifestSlice[]
+	[LEGACY_ROWS_PER_SLICE_KEY]?: unknown
+}
+
+/**
+ * A base manifest's slice list under either key.
+ *
+ * The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer
+ * measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the
+ * loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all.
+ */
+export function baseManifestSlices(base: BaseManifest): ManifestSlice[] {
+	const slices = base.slices?.length ? base.slices : base[LEGACY_SLICES_KEY]
+
+	if (slices?.length) return slices
+
+	throw new Error(
+		`base manifest lists no slices under "slices" or the pre-rename key — refusing to assemble an overlay whose base would be empty`
+	)
+}
+
+/**
+ * The base's rows-per-slice under either key, carried through to the new manifest verbatim.
+ */
+function baseRowsPerSlice(base: BaseManifest): unknown {
+	return base.rows_per_slice ?? base[LEGACY_ROWS_PER_SLICE_KEY]
+}
+
+/**
+ * One parquet to add, with the source label its rows carry.
+ */
+export interface OverlaySlice {
+	parquet: string
+	source: string
 }
 
 async function descriptor(
@@ -82,8 +132,10 @@ export interface OverlayManifestOptions {
 	newDir: string
 	modalRoot: string
 	version: string
-	sliceParquet: string
-	source: string
+	/**
+	 * The parquets this overlay adds, in the order they should appear after the base's own.
+	 */
+	slices: readonly OverlaySlice[]
 	note: string
 }
 
@@ -106,43 +158,58 @@ export function rerootBaseSlicePath(path: string, baseManifestPath: string): str
 }
 
 export async function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void> {
+	if (!args.slices.length) throw new Error("an overlay must add at least one slice")
+
 	const base = await readLocalJSONFile<BaseManifest>(args.base)
+	const baseSlices = baseManifestSlices(base)
 
-	if (base.slices.some((s) => s.source === args.source)) {
-		console.log(`WARN: base already contains source '${args.source}' — is this the right base?`)
+	for (const slice of args.slices) {
+		if (baseSlices.some((s) => s.source === slice.source)) {
+			console.log(`WARN: base already contains source '${slice.source}' — is this the right base?`)
+		}
 	}
 
-	const kept = base.slices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }))
+	const kept = baseSlices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }))
+	const added: SliceDescriptor[] = []
+
+	for (const slice of args.slices) {
+		added.push(
+			await descriptor(
+				join(args.newDir, "train", slice.parquet),
+				`${args.modalRoot}/train/${slice.parquet}`,
+				"train",
+				slice.source
+			)
+		)
+	}
 
-	const newTrain = await descriptor(
-		join(args.newDir, "train", args.sliceParquet),
-		`${args.modalRoot}/train/${args.sliceParquet}`,
-		"train",
-		args.source
-	)
+	const addedRows = added.reduce((total, slice) => total + slice.rows, 0)
+	const sources = args.slices.map((slice) => slice.source).join(", ")
 
 	const manifest = {
 		corpus_version: args.version,
 		overlay_base: base.corpus_version ?? null,
-		note:
-			args.note || `${base.corpus_version} slices (all kept verbatim) + the ${args.source} slice. Pure overlay add.`,
+		note: args.note || `${base.corpus_version} slices (all kept verbatim) + ${sources}. Pure overlay add.`,
 		schema: base.schema,
-		rows_per_slice: base.rows_per_slice,
+		rows_per_slice: baseRowsPerSlice(base),
 		row_group_size: base.row_group_size,
-		slices: [...kept, newTrain],
+		slices: [...kept, ...added],
 		counts: {
-			train: base.counts.train + newTrain.rows,
+			train: base.counts.train + addedRows,
 			val: base.counts.val,
 			test: base.counts.test,
 		},
-		total_rows: base.total_rows + newTrain.rows,
+		total_rows: base.total_rows + addedRows,
 	}
 
 	const out = join(args.newDir, "MANIFEST.json")
 	await writeLocalJSONFile(manifest, out)
 
 	console.log(`wrote ${out}`)
-	console.log(`  slices: ${manifest.slices.length} (${kept.length} base kept, +1 ${args.source})`)
+	console.log(`  slices: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`)
 	console.log(`  counts: ${JSON.stringify(manifest.counts)}  total: ${manifest.total_rows}`)
-	console.log(`  ${args.source} train: ${newTrain.rows} rows (${newTrain.bytes} bytes)`)
+
+	for (const slice of added) {
+		console.log(`  ${slice.source} train: ${slice.rows} rows (${slice.bytes} bytes)`)
+	}
 }

package/lib/tools/overture-subvenue.ts

@@ -40,7 +40,7 @@
 
 import { DatabaseClient } from "@mailwoman/sqlite/client"
 
-import type { SubVenueHarvestRow } from "#tools/sub-venue-lexicon"
+import type { SubVenueHarvestRow } from "#tools/sub/venue/lexicon"
 
 /**
  * Overture Places category → the designator its rows attest.