@mailwoman/corpus 8.5.0 → 9.0.0

package/src/shard-recipes/fr-order.ts

@@ -33,11 +33,12 @@
 import { spawnSync } from "node:child_process"
 
 import type { ComponentTag } from "@mailwoman/core/types"
+import { dataRootPath } from "@mailwoman/core/utils"
 
 import { stableSourceID } from "../adapter.ts"
 import { alignRow } from "../align.ts"
 import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
@@ -45,7 +46,7 @@ import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
    would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
    extracted as named constants above. */
 
-const SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
+const SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
 
 /**
  * Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
@@ -71,47 +72,9 @@ interface FrTuple {
 }
 
 /**
- * Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping).
- */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
-
-/**
- * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` to stay under V8's
- * string limit. Only keeps rows with a house_number (the shard's core signal) and a postcode (required for
- * reversed-order rendering to be meaningful).
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` so the whole file
+ * never has to be held in memory. Only keeps rows with a house_number (the shard's core signal) and a postcode
+ * (required for reversed-order rendering to be meaningful).
  */
 function readTuples(limit: number): FrTuple[] {
 	const maxLines = Math.max(limit * 8, 40_000) + 1
@@ -127,28 +90,15 @@ function readTuples(limit: number): FrTuple[] {
 		return []
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (name: string): number => header.indexOf(name)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iCity = idx("city"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: FrTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length && tuples.length < limit; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-		const street = get(cells, iStreet)
-		const locality = get(cells, iCity)
-		const house_number = get(cells, iNum)
-		const postcode = get(cells, iPost)
+	for (const row of readCSVRecords(r.stdout)) {
+		if (tuples.length >= limit) break
+		const street = row.street ?? ""
+		const locality = row.city ?? ""
+		const house_number = row.number ?? ""
+		const postcode = row.postcode ?? ""
 
 		// Require all four fields: HN is the signal; postcode drives reversed-order variants.
 		if (!street || !locality || !house_number || !postcode) continue
@@ -216,7 +166,7 @@ export const frOrderRecipe: ShardRecipe = {
 		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
 
 		if (!pool.length) {
-			throw new Error("No FR tuples found — is /tmp/oa-cache/fr__countrywide.zip present?")
+			throw new Error(`No FR tuples found — is ${SOURCE.zip} present?`)
 		}
 
 		let emitted = 0

package/src/shard-recipes/german.ts

@@ -17,10 +17,12 @@
 
 import { spawnSync } from "node:child_process"
 
+import { dataRootPath } from "@mailwoman/core/utils"
+
 import { stableSourceID } from "../adapter.ts"
 import { alignRow } from "../align.ts"
 import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /**
  * A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
@@ -37,48 +39,10 @@ interface GermanSource {
  * Berlin (a city-state, region==locality); sn/statewide → Sachsen.
  */
 const SOURCES: GermanSource[] = [
-	{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
-	{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
+	{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
+	{ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
 ]
 
-/**
- * Minimal RFC-4180-ish splitter (handles quoted fields).
- */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]!
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
-
 /**
  * Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
  */
@@ -91,35 +55,18 @@ function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
 		return []
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (name: string): number => header.indexOf(name)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iCity = idx("city"),
-		iRegion = idx("region"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: LocaleBaseTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length; li++) {
-		const lineStr = lines[li]
-
-		if (!lineStr) continue
-		const cells = splitCSV(lineStr)
-		const street = get(cells, iStreet)
-		const locality = get(cells, iCity)
+	for (const row of readCSVRecords(r.stdout)) {
+		const street = row.street ?? ""
+		const locality = row.city ?? ""
 
 		if (!street || !locality) continue
-		const house_number = get(cells, iNum)
-		const postcode = get(cells, iPost)
+		const house_number = row.number ?? ""
+		const postcode = row.postcode ?? ""
 		// OA's REGION column is empty for DE — fall back to the source's Bundesland (set per file).
-		const region = get(cells, iRegion) || source.region || ""
+		const region = row.region || source.region || ""
 		const key = `${house_number}|${street}|${locality}|${postcode}`.toLowerCase()
 
 		if (seen.has(key)) continue
@@ -165,7 +112,7 @@ export const germanRecipe: ShardRecipe = {
 		}
 
 		if (!pool.length) {
-			throw new Error("No German tuples found — are the cached zips present in /tmp/oa-cache?")
+			throw new Error(`No German tuples found — are the cached zips present in ${dataRootPath("oa-cache")}?`)
 		}
 
 		let emitted = 0

package/src/shard-recipes/index.ts

@@ -32,6 +32,7 @@ import { siBareVillageRecipe } from "./si-bare-village.ts"
 import { streetAffixRecipe } from "./street-affix.ts"
 import { streetBareRecipe } from "./street-bare.ts"
 import { streetRecipe } from "./street.ts"
+import { subVenueRecipe } from "./sub-venue.ts"
 import { unitRecipe } from "./unit.ts"
 
 export * from "./scaffold.ts"
@@ -64,6 +65,7 @@ const RECIPES: readonly ShardRecipe[] = [
 	countryBalancedRecipe,
 	boundaryStressRecipe,
 	anchorAbsorptionRecipe,
+	subVenueRecipe,
 ]
 
 /**

package/src/shard-recipes/intersection.ts

@@ -30,8 +30,8 @@
  *   External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
  *
  *   - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
- *       /tmp/tiger-edges)
- *   - /tmp/oa-cache/us__il__cook.zip (ZIP→city tails)
+ *       `$MAILWOMAN_DATA_ROOT/census/tiger2023-edges`, where `mailwoman situs interpolation` puts them)
+ *   - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
  */
 
 import { spawnSync } from "node:child_process"
@@ -39,12 +39,13 @@ import { existsSync, readFileSync, writeFileSync } from "node:fs"
 
 import type { DuckDBConnection } from "@duckdb/node-api"
 import type { ComponentTag } from "@mailwoman/core/types"
-import { repoRootPath } from "@mailwoman/core/utils"
+import { dataRootPath, repoRootPath } from "@mailwoman/core/utils"
+import { JSONSpliterator } from "spliterator"
 
 import { stableSourceID } from "../adapter.ts"
 import { alignRow } from "../align.ts"
 import type { CanonicalRow, LabeledRow } from "../types.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 interface County {
 	fips: string
@@ -60,7 +61,13 @@ const TRAIN_COUNTIES: readonly County[] = [
 const GOLDEN_COUNTIES: readonly County[] = [{ fips: "50023", state: "VT", regime: "rural" }]
 
 const EVAL_GOLD_PATH = repoRootPath("data", "eval", "external", "intersection-real.jsonl")
-const OA_COOK = { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv" }
+/**
+ * Where `mailwoman situs interpolation` unpacks the per-county TIGER EDGES shapefiles — a Census download, so it sits
+ * beside the other Census vintages under `census/`. `--edges-dir` overrides it.
+ */
+const DEFAULT_EDGES_DIR = dataRootPath("census", "tiger2023-edges")
+
+const OA_COOK = { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv" }
 
 /**
  * One real crossing extracted from a county's TIGER EDGES shapefile.
@@ -157,52 +164,22 @@ function weightedPick<T extends { w: number }>(items: readonly T[], random: () =
 }
 
 /**
- * Minimal RFC-4180-ish splitter (handles quoted fields) — same as the affix builder.
+ * Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
  */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
+const pairKey = (a: string, b: string): string => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u001F")
 
 /**
- * Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
+ * One row of the intersection eval gold — the shape this recipe reads to exclude eval crossings.
  */
-const pairKey = (a: string, b: string): string => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u001F")
+interface EvalGoldRow {
+	node: number | string
+	components: { intersection_a: string; intersection_b: string }
+}
 
 /**
  * Load the eval's crossings so neither train nor golden ever sees them.
  */
-function readEvalExclusions(): { nodes: Set<number>; pairs: Set<string> } {
+async function readEvalExclusions(): Promise<{ nodes: Set<number>; pairs: Set<string> }> {
 	const nodes = new Set<number>()
 	const pairs = new Set<string>()
 
@@ -212,9 +189,7 @@ function readEvalExclusions(): { nodes: Set<number>; pairs: Set<string> } {
 		return { nodes, pairs }
 	}
 
-	for (const line of readFileSync(EVAL_GOLD_PATH, "utf8").split("\n")) {
-		if (!line) continue
-		const row = JSON.parse(line)
+	for await (const row of JSONSpliterator.fromAsync<EvalGoldRow>(EVAL_GOLD_PATH)) {
 		nodes.add(Number(row.node))
 		pairs.add(pairKey(row.components.intersection_a, row.components.intersection_b))
 	}
@@ -285,20 +260,12 @@ function buildZipCityMap(): Map<string, string> {
 		return new Map()
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return new Map()
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const iCity = header.indexOf("city")
-	const iPost = header.indexOf("postcode")
 	const counts = new Map<string, Map<string, number>>()
 
 	// zip → Map(city → n)
-	for (let li = 1; li < lines.length; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-		const city = (cells[iCity] ?? "").trim()
-		const zip = (cells[iPost] ?? "").trim()
+	for (const row of readCSVRecords(r.stdout)) {
+		const city = row.city ?? ""
+		const zip = row.postcode ?? ""
 
 		if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city)) continue
 		let byCity = counts.get(zip)
@@ -486,15 +453,20 @@ export const intersectionRecipe: ShardRecipe = {
 	name: "intersection",
 	description: "Real-pair intersection rows (US): TIGER 2023 EDGES crossings → audited intersection_a/b labels",
 	mode: "generate",
-	options: [{ flag: "--edges-dir <dir>", description: "Unzipped TIGER 2023 EDGES dir. Default /tmp/tiger-edges" }],
+	options: [
+		{
+			flag: "--edges-dir <dir>",
+			description: `Unzipped TIGER 2023 EDGES dir. Default ${DEFAULT_EDGES_DIR}`,
+		},
+	],
 	async run(opts, write) {
 		// Legacy build-intersection-shard.mjs seeded `mulberry32(opts.seed)`.
 		const random = makeMulberry32(opts.seed)
 		const count = opts.count ?? 40_000
 		const source = opts.sourceName ?? "synth-intersection"
-		const edgesDir = opts.edgesDir ?? "/tmp/tiger-edges"
+		const edgesDir = opts.edgesDir ?? DEFAULT_EDGES_DIR
 		const counties = opts.golden ? GOLDEN_COUNTIES : TRAIN_COUNTIES
-		const exclusions = readEvalExclusions()
+		const exclusions = await readEvalExclusions()
 
 		console.error(`  eval exclusions: ${exclusions.nodes.size} nodes, ${exclusions.pairs.size} pairs`)
 

package/src/shard-recipes/locale.ts

@@ -86,26 +86,26 @@ export interface LocaleCountrySource {
 }
 
 /**
- * Per-country OA sources + the source name used in the corpus. DE/FR still point at the legacy `/tmp/oa-cache` zips
- * (their historical build inputs — materialize them there to regenerate). ES/NL read the extracted countrywide CSVs and
- * IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is OA-conformed, so the old
- * raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is empty; the
- * international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad autónoma, IT =
- * regione, NL = province).
+ * Per-country OA sources + the source name used in the corpus. DE/FR read their historical build inputs, the cached
+ * zips under `$MAILWOMAN_DATA_ROOT/oa-cache` — materialize them there to regenerate. ES/NL read the extracted
+ * countrywide CSVs and IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is
+ * OA-conformed, so the old raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is
+ * empty; the international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad
+ * autónoma, IT = regione, NL = province).
  */
 const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
 	DE: {
 		source: "synth-german",
 		corpusVersion: "0.4.0",
 		parts: [
-			{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
-			{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
+			{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
+			{ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
 		],
 	},
 	FR: {
 		source: "synth-fr",
 		corpusVersion: "0.4.0",
-		parts: [{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }],
+		parts: [{ zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }],
 	},
 	NL: {
 		source: "synth-nl",
@@ -144,7 +144,7 @@ const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
 		parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
 	},
 	GB: {
-		// HM Land Registry Price Paid Data tuples (25.67M rows; see Task 2's ppd ingest). PPD's DISTRICT is the
+		// HM Land Registry Price Paid Data tuples (25.67M rows out of the PPD ingest). PPD's DISTRICT is the
 		// postal town (locality) and CITY is the dependent locality — legitimately EMPTY on the majority of rows
 		// (most GB addresses have no dependent locality). `districtAsLocality` maps DISTRICT→locality and, when
 		// present, CITY→dependent_locality; the `readTuples` gate above only drops a row when BOTH are empty, so
@@ -256,9 +256,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
 
 	try {
 		// CSVSpliterator handles OA's quoted fields (embedded commas/newlines) and CRLF row terminators
-		// (spliterator ≥ 3.2.0); `header: false` yields the header row too, so we build the column index
-		// from it exactly as the prior hand-rolled split did. Verified byte-identical tuples vs that split
-		// on 10M+ real OA rows — the only raw-cell difference was a trailing CR on the discarded HASH column.
+		// (spliterator ≥ 3.2.0); `header: false` yields the header row too, so the column index is built
+		// from that first row rather than from a header option.
 		for await (const cells of CSVSpliterator.fromAsync<string[]>(input, {
 			mode: "array",
 			header: false,
@@ -315,9 +314,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
 			// dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
 			// sub-locality. GB PPD tuples flip which side is legitimately empty — on the MAJORITY of GB rows CITY
 			// (the dependent_locality) is empty and DISTRICT (the locality) is populated, so the gate below only
-			// drops a `districtAsLocality` row when BOTH are empty, not when CITY alone is (that used to silently
-			// drop most of the GB source — see the fixed `readTuples` gate below). See
-			// {@link LocalePart.districtAsLocality}.
+			// drops a `districtAsLocality` row when BOTH are empty, never when CITY alone is — gating on CITY
+			// alone silently discards most of the GB source. See {@link LocalePart.districtAsLocality}.
 			let locality: string | null
 			let dependent_locality: string | undefined
 

package/src/shard-recipes/no-fragment.ts

@@ -35,7 +35,7 @@
  *   list. Diacritic-KEEPING normalizer, matching the board (see the norm docstring).
  */
 
-import { readFileSync } from "node:fs"
+import { TextSpliterator } from "spliterator"
 
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
@@ -100,12 +100,15 @@ export const noFragmentRecipe: ShardRecipe = {
 			)
 		}
 
-		const excluded = new Set(
-			readFileSync(excludePath, "utf8")
-				.split("\n")
-				.map((line) => line.trim())
-				.filter((line) => line && !line.startsWith("#"))
-		)
+		const excluded = new Set<string>()
+
+		for await (const line of TextSpliterator.fromAsync(excludePath)) {
+			const trimmed = line.trim()
+
+			if (trimmed && !trimmed.startsWith("#")) {
+				excluded.add(trimmed)
+			}
+		}
 
 		if (!excluded.size) throw new Error(`no-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)
 

package/src/shard-recipes/no-street-led.ts

@@ -16,7 +16,7 @@
  *   3. pc-first   "«pc» «city», «st» «n»"   (the 7% floor — the anchor the others converge to)
  */
 
-import { readFileSync } from "node:fs"
+import { TextSpliterator } from "spliterator"
 
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
@@ -62,12 +62,15 @@ export const noStreetLedRecipe: ShardRecipe = {
 			)
 		}
 
-		const excluded = new Set(
-			readFileSync(excludePath, "utf8")
-				.split("\n")
-				.map((line) => line.trim())
-				.filter((line) => line && !line.startsWith("#"))
-		)
+		const excluded = new Set<string>()
+
+		for await (const line of TextSpliterator.fromAsync(excludePath)) {
+			const trimmed = line.trim()
+
+			if (trimmed && !trimmed.startsWith("#")) {
+				excluded.add(trimmed)
+			}
+		}
 
 		if (!excluded.size) throw new Error(`no-street-led: --exclude-surfaces "${excludePath}" listed no surfaces`)