@mailwoman/corpus 8.5.0 → 9.0.0

package/src/shard-recipes/po-box-cedex.ts

@@ -20,9 +20,9 @@
  *   locality-hash holdout (hash%10===0).
  *
  *   Prerequisites (read once, before the generation loop — these do NOT consume `random`): the cached
- *   OA zips in /tmp/oa-cache, the GeoNames Canada dump at /tmp/geonames-cache/CA.zip, and the
- *   GeoNames POSTAL-CODE dumps for AU/NZ (/tmp/geonames-cache/{AU,NZ}-postal.zip). See the legacy
- *   script header for the exact curl commands.
+ *   OA zips in `$MAILWOMAN_DATA_ROOT/oa-cache`, the GeoNames Canada dump at
+ *   `$MAILWOMAN_DATA_ROOT/geonames/CA.zip`, and the GeoNames POSTAL-CODE dumps for AU/NZ
+ *   (`$MAILWOMAN_DATA_ROOT/geonames-postal/{AU,NZ}.zip`).
  *
  *   Byte-fidelity: the legacy script seeded its own mulberry32 from `--seed`
  *   (`mulberry32(opts.seed)`); this recipe re-creates the SAME generator
@@ -35,8 +35,10 @@ import { spawnSync } from "node:child_process"
 import { isAuDeliveryService, isAuPostcode, isAuStateAbbreviation } from "@mailwoman/codex/au"
 import { FSA_LETTER_TO_PROVINCE, normalizeCaPostalCode } from "@mailwoman/codex/ca"
 import { isCedex } from "@mailwoman/codex/fr"
-import { isNzDeliveryService, isNzPostcode } from "@mailwoman/codex/nz"
+import { isNZDeliveryService, isNZPostcode } from "@mailwoman/codex/nz"
 import { isPOBox } from "@mailwoman/codex/us"
+import { dataRootPath } from "@mailwoman/core/utils"
+import { TextSpliterator } from "spliterator"
 
 import { alignRow } from "../align.ts"
 import {
@@ -45,7 +47,7 @@ import {
 	synthesizeMilitaryPoBoxRow,
 	type LocaleTemplate,
 } from "../synthesize-po-box.ts"
-import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readCSVRecords, shardSourceID, type CanonicalShardRow, type ShardRecipe } from "./scaffold.ts"
 
 // ── Base-skeleton sources ────────────────────────────────────────────────────────────────────────
 // Same OA cache as the unit/affix shards. US train = every NON-Vermont state; US eval = Vermont (the
@@ -58,20 +60,29 @@ import { makeMulberry32, shardSourceID, type CanonicalShardRow, type ShardRecipe
    extracted as named constants above. */
 
 const US_TRAIN_SOURCES = [
-	{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
-	{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
-	{ zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
-	{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
-	{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
-	{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
-	{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
+	{ zip: dataRootPath("oa-cache", "us__ca__berkeley.zip"), csv: "us/ca/berkeley.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__ca__marin.zip"), csv: "us/ca/marin.csv", region: "CA" },
+	{ zip: dataRootPath("oa-cache", "us__dc__statewide.zip"), csv: "us/dc/statewide.csv", region: "DC" },
+	{ zip: dataRootPath("oa-cache", "us__ia__statewide.zip"), csv: "us/ia/statewide.csv", region: "IA" },
+	{ zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv", region: "IL" },
+	{ zip: dataRootPath("oa-cache", "us__mt__statewide.zip"), csv: "us/mt/statewide.csv", region: "MT" },
+	{ zip: dataRootPath("oa-cache", "us__sd__statewide.zip"), csv: "us/sd/statewide.csv", region: "SD" },
 ]
 
-const US_EVAL_SOURCE = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" }
-const FR_SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
-const GEONAMES_CA = "/tmp/geonames-cache/CA.zip"
-const GEONAMES_POSTAL_AU = { zip: "/tmp/geonames-cache/AU-postal.zip", txt: "AU.txt" }
-const GEONAMES_POSTAL_NZ = { zip: "/tmp/geonames-cache/NZ-postal.zip", txt: "NZ.txt" }
+const US_EVAL_SOURCE = {
+	zip: dataRootPath("oa-cache", "us__vt__statewide.zip"),
+	csv: "us/vt/statewide.csv",
+	region: "VT",
+}
+
+const FR_SOURCE = { zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }
+// GeoNames has two per-country exports with different schemas, and this recipe reads both. The MAIN
+// dump (feature class + population columns) lives under `geonames/`; the POSTAL-CODE dump lives
+// under `geonames-postal/`, where the directory carries the "postal" distinction the filename used
+// to.
+const GEONAMES_CA = dataRootPath("geonames", "CA.zip")
+const GEONAMES_POSTAL_AU = { zip: dataRootPath("geonames-postal", "AU.zip"), txt: "AU.txt" }
+const GEONAMES_POSTAL_NZ = { zip: dataRootPath("geonames-postal", "NZ.zip"), txt: "NZ.txt" }
 
 /**
  * ── Surface vocabulary (codex + corpus templates — see the header) ──────────────────────────────.
@@ -172,20 +183,20 @@ interface USTuple {
 	postcode: string
 }
 
-interface FrTuple {
+interface FRTuple {
 	house_number: string
 	street: string
 	locality: string
 	postcode: string
 }
 
-interface AuTuple {
+interface AUTuple {
 	locality: string
 	region: string
 	postcode: string
 }
 
-interface NzTuple {
+interface NZTuple {
 	locality: string
 	postcode: string
 }
@@ -214,44 +225,6 @@ function localityHash(name: string): number {
 
 const isHoldoutLocality = (name: string): boolean => localityHash(name) % 10 === 0
 
-/**
- * Minimal RFC-4180-ish splitter (handles quoted fields).
- */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]!
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
-
 const MAX_LOCALITY_LENGTH = 40
 
 const cleanLocality = (loc: string) =>
@@ -269,38 +242,24 @@ function readUsTuples(source: { zip: string; csv: string; region: string }): UST
 		return []
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (name: string) => header.indexOf(name)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iCity = idx("city"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: USTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-		const locality = get(cells, iCity)
+	for (const row of readCSVRecords(r.stdout)) {
+		const locality = row.city ?? ""
 
 		if (!cleanLocality(locality)) continue
 		const key = locality.toLowerCase()
 
-		const street = get(cells, iStreet),
-			house_number = get(cells, iNum)
+		const street = row.street ?? "",
+			house_number = row.number ?? ""
 
 		// One tuple per (locality, street) pair keeps the pool varied without ballooning memory.
 		const pairKey = `${key}|${street}`.toLowerCase()
 
 		if (seen.has(pairKey)) continue
 		seen.add(pairKey)
-		tuples.push({ house_number, street, locality, region: source.region, postcode: get(cells, iPost) })
+		tuples.push({ house_number, street, locality, region: source.region, postcode: row.postcode ?? "" })
 	}
 
 	return tuples
@@ -308,10 +267,13 @@ function readUsTuples(source: { zip: string; csv: string; region: string }): UST
 
 /**
  * Stride-sampled FR tuples (number/street/city/postcode). The countrywide CSV is 2.5 GB and insee-ordered; `awk NR%K`
- * strides the whole country instead of reading one département. Quoted commas survive because awk only FILTERS lines —
- * parsing stays in splitCSV.
+ * strides the whole country instead of reading one département.
+ *
+ * The stride counts PHYSICAL lines, so a record carrying a newline inside a quoted field is two lines to awk and can be
+ * cut in half by the stride. That is a sampling artefact of the awk pre-filter, not of the parse: whichever lines
+ * survive are re-assembled into records by {@link readCSVRecords}, and a halved record fails the field checks below.
  */
-function readFrTuples(limit: number): FrTuple[] {
+function readFrTuples(limit: number): FRTuple[] {
 	const r = spawnSync(
 		"bash",
 		["-c", `unzip -p "${FR_SOURCE.zip}" "${FR_SOURCE.csv}" | awk 'NR==1 || NR%211==3' | head -n ${limit + 1}`],
@@ -324,29 +286,14 @@ function readFrTuples(limit: number): FrTuple[] {
 		return []
 	}
 
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return []
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const idx = (n: string) => header.indexOf(n)
-
-	const iNum = idx("number"),
-		iStreet = idx("street"),
-		iCity = idx("city"),
-		iPost = idx("postcode")
-
-	const get = (cells: string[], i: number) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
-	const tuples: FrTuple[] = []
+	const tuples: FRTuple[] = []
 	const seen = new Set<string>()
 
-	for (let li = 1; li < lines.length; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-
-		const locality = get(cells, iCity),
-			postcode = get(cells, iPost),
-			street = get(cells, iStreet),
-			house_number = get(cells, iNum)
+	for (const row of readCSVRecords(r.stdout)) {
+		const locality = row.city ?? "",
+			postcode = row.postcode ?? "",
+			street = row.street ?? "",
+			house_number = row.number ?? ""
 
 		if (!cleanLocality(locality) || !/^\d{5}$/.test(postcode) || !street || !house_number) continue
 		const key = `${locality}|${street}`.toLowerCase()
@@ -379,7 +326,7 @@ function readCaLocalities(admin1: string): string[] {
 		return []
 	}
 
-	return [...new Set(r.stdout.split("\n").filter(cleanLocality))]
+	return [...new Set([...TextSpliterator.from(r.stdout)].filter(cleanLocality))]
 }
 
 /**
@@ -392,7 +339,7 @@ function readCaLocalities(admin1: string): string[] {
 function readPostalTuples(
 	source: { zip: string; txt: string },
 	opts: { withState: boolean }
-): Array<AuTuple | NzTuple> {
+): Array<AUTuple | NZTuple> {
 	const r = spawnSync("unzip", ["-p", source.zip, source.txt], { maxBuffer: 1024 * 1024 * 64, encoding: "utf8" })
 
 	if (r.status !== 0) {
@@ -401,12 +348,14 @@ function readPostalTuples(
 		return []
 	}
 
-	const tuples: Array<AuTuple | NzTuple> = []
+	const tuples: Array<AUTuple | NZTuple> = []
 	const seen = new Set<string>()
-	const validPostcode = opts.withState ? isAuPostcode : isNzPostcode
+	const validPostcode = opts.withState ? isAuPostcode : isNZPostcode
 
-	for (const line of r.stdout.split("\n")) {
+	for (const line of TextSpliterator.from(r.stdout)) {
 		if (!line) continue
+
+		// oxlint-disable-next-line mailwoman/prefer-spliterator -- One row already streamed off the line above.
 		const cols = line.split("\t")
 		const postcode = (cols[1] ?? "").trim()
 		const locality = (cols[2] ?? "").trim()
@@ -514,7 +463,7 @@ function makeCedex(random: () => number): string {
 /**
  * Compose an AU or NZ delivery-service phrase from the codex-sourced leaders. Same contract as makePoBoxPhrase: a
  * phrase built from a codex-known designator and a clean id must round-trip the codex matcher (isAuDeliveryService /
- * isNzDeliveryService) — a failure is a generation bug, loud. Noisy ids (commas / embedded spaces) are corpus-designed
+ * isNZDeliveryService) — a failure is a generation bug, loud. Noisy ids (commas / embedded spaces) are corpus-designed
  * adversarial forms, exempt.
  */
 function makeAuNzPoBoxPhrase(
@@ -532,14 +481,14 @@ function makeAuNzPoBoxPhrase(
 	let num = maybeNoisifyBoxNumber(pickBoxNumber(random), random)
 
 	// NZ CMB identifiers are alpha-led per the ADV358 example ("CMB B99").
-	if (leader === "CMB" && validate === isNzDeliveryService) {
+	if (leader === "CMB" && validate === isNZDeliveryService) {
 		num = `B${num}`
 	}
 
 	const phrase = `${caseDial(random, leader)} ${num}`
 	// The "clean id" shape differs per system: ADV358 identifiers carry no separators at all, the AU
 	// AMAS id (like the US one) tolerates dashes. Noisy ids outside the clean shape are exempt.
-	const cleanID = validate === isNzDeliveryService ? /^[\dA-Za-z]+$/ : /^[\dA-Za-z][\dA-Za-z-]*$/
+	const cleanID = validate === isNZDeliveryService ? /^[\dA-Za-z]+$/ : /^[\dA-Za-z][\dA-Za-z-]*$/
 
 	if (cleanID.test(num) && !validate(phrase)) {
 		throw new Error(`generated a phrase the codex matcher rejects: "${phrase}"`)
@@ -614,7 +563,7 @@ function renderPmbUs(random: () => number, t: USTuple): Rendered {
 	return { fmt: "pmb-bare", raw: `${road} ${phrase}`, components: { house_number: hn, street, po_box: phrase } }
 }
 
-function renderBpFr(random: () => number, t: FrTuple): Rendered {
+function renderBpFr(random: () => number, t: FRTuple): Rendered {
 	const phrase = makePoBoxPhrase(random, FR_LEADERS)
 	const { locality, postcode: pc } = t
 	const upper = random() < 0.5
@@ -651,7 +600,7 @@ function renderBpFr(random: () => number, t: FrTuple): Rendered {
 	}
 }
 
-function renderCedexFr(random: () => number, t: FrTuple): Rendered {
+function renderCedexFr(random: () => number, t: FRTuple): Rendered {
 	const cedex = makeCedex(random)
 	const { house_number: hn, street, locality, postcode: pc } = t
 	const loc = random() < 0.6 ? locality.toUpperCase() : locality
@@ -703,7 +652,7 @@ function renderCaEn(random: () => number, loc: string): Rendered {
 	return { fmt: "ca-en-bare", raw: phrase, components: { po_box: phrase } }
 }
 
-function renderAuPoBox(random: () => number, t: AuTuple): Rendered {
+function renderAUPoBox(random: () => number, t: AUTuple): Rendered {
 	const phrase = makeAuNzPoBoxPhrase(random, AU_LEADERS_CURRENT, AU_LEADERS_LEGACY, isAuDeliveryService)
 	const { locality, region: reg, postcode: pc } = t
 	const r = random()
@@ -737,8 +686,8 @@ function renderAuPoBox(random: () => number, t: AuTuple): Rendered {
 	return { fmt: "au-venue", raw: `${v}, ${phrase}, ${loc} ${reg} ${pc}`, components: { venue: v, ...base } }
 }
 
-function renderNzPoBox(random: () => number, t: NzTuple): Rendered {
-	const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE, isNzDeliveryService)
+function renderNZPoBox(random: () => number, t: NZTuple): Rendered {
+	const phrase = makeAuNzPoBoxPhrase(random, NZ_LEADERS_COMMON, NZ_LEADERS_RARE, isNZDeliveryService)
 	const { locality, postcode: pc } = t
 	// NZ addresses are written mixed-case ("PO Box 4099, Timaru 7942") — no region line (ADV358).
 	const base = { po_box: phrase, locality, postcode: pc }
@@ -818,8 +767,8 @@ export const poBoxCedexRecipe: ShardRecipe = {
 		console.error(`  GeoNames CA: QC ${qcAll.length}→${qcPool.length}, ON ${onAll.length}→${onPool.length}`)
 
 		// AU/NZ pools: same stable locality-hash holdout as FR/CA.
-		const auAll = readPostalTuples(GEONAMES_POSTAL_AU, { withState: true }) as AuTuple[]
-		const nzAll = readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false }) as NzTuple[]
+		const auAll = readPostalTuples(GEONAMES_POSTAL_AU, { withState: true }) as AUTuple[]
+		const nzAll = readPostalTuples(GEONAMES_POSTAL_NZ, { withState: false }) as NZTuple[]
 		const auPool = auAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
 		const nzPool = nzAll.filter((t) => isHoldoutLocality(t.locality) === opts.golden)
 
@@ -827,7 +776,7 @@ export const poBoxCedexRecipe: ShardRecipe = {
 
 		if (!usPool.length || !frPool.length || !qcPool.length || !onPool.length || !auPool.length || !nzPool.length) {
 			throw new Error(
-				"A base pool is empty — check /tmp/oa-cache and /tmp/geonames-cache (CA.zip, AU-postal.zip, NZ-postal.zip)."
+				`A base pool is empty — check ${dataRootPath("oa-cache")}, ${GEONAMES_CA}, ${GEONAMES_POSTAL_AU.zip}, and ${GEONAMES_POSTAL_NZ.zip}.`
 			)
 		}
 
@@ -877,11 +826,11 @@ export const poBoxCedexRecipe: ShardRecipe = {
 				country = "CA"
 				locale = "fr-CA"
 			} else if (cls === "po-box-au") {
-				rendered = renderAuPoBox(random, pick(random, auPool))
+				rendered = renderAUPoBox(random, pick(random, auPool))
 				country = "AU"
 				locale = "en-AU"
 			} else if (cls === "po-box-nz") {
-				rendered = renderNzPoBox(random, pick(random, nzPool))
+				rendered = renderNZPoBox(random, pick(random, nzPool))
 				country = "NZ"
 				locale = "en-NZ"
 			} else if (cls === "po-box-us-military") {

package/src/shard-recipes/scaffold.ts

@@ -9,6 +9,9 @@
  *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
  */
 
+import { tryParsingJSON } from "@mailwoman/core/objects"
+import { CSVSpliterator } from "spliterator"
+
 import { stableSourceID } from "../adapter.ts"
 import { alignRow } from "../align.ts"
 
@@ -33,38 +36,65 @@ export interface ShardTuple {
 }
 
 /**
- * The deterministic LCG (`s = s*1664525 + 1013904223 mod 2^32`) the street/po-box/anchor builders used. A recipe whose
- * legacy `.mjs` seeded this must create it here so `--seed N` is byte-reproducible.
+ * The two seeded generators the legacy `build-*-shard` scripts used, re-exported from their home in
+ * `@mailwoman/core/utils` so a recipe keeps importing everything it needs from this one scaffold module.
+ *
+ * - `makeLcg` (`s = s*1664525 + 1013904223 mod 2^32`) — what the street/po-box/anchor builders seeded.
+ * - `makeMulberry32` — what the MAJORITY of them used (german, locale, boundary-stress, unit, fr-order, country-balanced,
+ *   intersection, fr-admin-split, street-affix, street-bare, po-box-cedex).
+ *
+ * A recipe must seed the same one its `.mjs` did, the same way it did (usually `seed`, but some derive a per-stream
+ * seed), or `--seed N` stops being byte-reproducible.
  */
-export function makeLcg(seed: number): () => number {
-	let s = seed >>> 0
-
-	return () => {
-		s = (s * 1_664_525 + 1_013_904_223) % 4_294_967_296
-
-		return s / 4_294_967_296
-	}
-}
+export { makeLcg, mulberry32 as makeMulberry32, makeLcg as makeRandom } from "@mailwoman/core/utils"
 
 /**
- * Back-compat alias for {@link makeLcg}.
+ * One CSV record, keyed by its lower-cased header name, every value trimmed. A column the header does not declare reads
+ * as `undefined`; a column the header declares but this record does not reach reads as `""`.
  */
-export const makeRandom = makeLcg
+export type CSVRecord = Record<string, string | undefined>
 
 /**
- * Mulberry32 — the PRNG the MAJORITY of the legacy `build-*-shard` scripts used (german, locale, boundary-stress, unit,
- * fr-order, country-balanced, intersection, fr-admin-split, street-affix, street-bare, po-box-cedex). A recipe must
- * seed it EXACTLY as its `.mjs` did (usually `seed`, but some derive a per-stream seed) to stay byte-reproducible.
+ * Read an in-memory CSV (an `unzip -p` buffer, say) as header-keyed records.
+ *
+ * Quote handling spans the ROW split, not only the column split: a newline inside a quoted field belongs to a single
+ * record, so the record boundaries can only be found by a scanner that already knows where the quotes are. Find the
+ * lines first and unquote afterwards and such a record splits in two — the first half short by however many columns
+ * followed the newline, which is how a `street` value comes to be read as `city`.
+ *
+ * Header names are lower-cased on the way in. `normalizeKeys` will not do it: it leaves an ALL CAPS header alone, and
+ * OpenAddresses ships `LON,LAT,NUMBER,STREET` while other extracts ship the same names lower-case. A recipe names its
+ * columns in lower case either way.
+ *
+ * Line breaks inside a value become single spaces. A quote-aware parse is the first thing here able to return a value
+ * CONTAINING one — `us/ia/statewide.csv` has 12, all unit designators like `"#2\n#2"` — and every consumer synthesizes
+ * one-line address text from these cells with no guard, because until that parse landed no value could carry one.
+ * Collapsing keeps the record (the address is fine; the source's line break is not part of it) without emitting a
+ * training row with a newline inside it.
+ *
+ * Only `\r` and `\n`, deliberately — NOT `\s`. Runs of spaces and tabs pass through exactly as the source wrote them
+ * (OA's IA extract writes `NORTH`, three spaces, `MAIN STREET`), because those could always appear and every shard
+ * built to date contains them. Widening this to `\s+` silently rewrites values on rows with no line break at all.
+ * `scaffold.test.ts` pins both halves.
  */
-export function makeMulberry32(seed: number): () => number {
-	let a = seed >>> 0
+export function* readCSVRecords(source: Uint8Array): Generator<CSVRecord> {
+	let header: string[] | null = null
+
+	// `header: false` keeps the first record in the stream so this reader owns the lower-casing.
+	for (const cells of CSVSpliterator.from(source, { header: false, enableQuoteHandling: true })) {
+		if (!header) {
+			header = cells.map((name) => name.trim().toLowerCase())
+
+			continue
+		}
 
-	return () => {
-		a = (a + 0x6d_2b_79_f5) | 0
-		let t = Math.imul(a ^ (a >>> 15), 1 | a)
-		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+		const record: CSVRecord = {}
 
-		return ((t ^ (t >>> 14)) >>> 0) / 4_294_967_296
+		for (let i = 0; i < header.length; i++) {
+			record[header[i]!] = (cells[i] ?? "").replaceAll(/[\r\n]+/g, " ").trim()
+		}
+
+		yield record
 	}
 }
 
@@ -73,7 +103,7 @@ export function makeMulberry32(seed: number): () => number {
  */
 export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
 	// TextSpliterator (not JSONSpliterator) so a malformed line is SKIPPED, not thrown — the
-	// per-line try/catch below is the tolerance this reader has always had.
+	// per-line tryParsingJSON below is the tolerance this reader has always had.
 	const { TextSpliterator } = await import("spliterator")
 
 	for await (const line of TextSpliterator.fromAsync(input)) {
@@ -81,10 +111,10 @@ export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
 
 		if (!trimmed) continue
 
-		try {
-			yield JSON.parse(trimmed) as ShardTuple
-		} catch {
-			// skip malformed line
+		const tuple = tryParsingJSON<ShardTuple>(trimmed)
+
+		if (tuple !== null) {
+			yield tuple
 		}
 	}
 }
@@ -178,6 +208,30 @@ export interface ShardRecipeOpts {
 	 * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3.
 	 */
 	longNumberMinDigits?: number
+	/**
+	 * `sub-venue`: the sub-venue lexicon JSON. Default = the committed `corpus/data/sub-venue-lexicon.json`, resolved
+	 * through the package manifest so it works from the source tree and from `out/`.
+	 */
+	lexicon?: string
+	/**
+	 * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region. Default
+	 * `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`.
+	 */
+	extractsDir?: string
+	/**
+	 * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound pools (the two of poi.db's
+	 * four countries this shard has legs for). Default `$MAILWOMAN_DATA_ROOT/poi/poi.db`.
+	 */
+	poiDb?: string
+	/**
+	 * `sub-venue`: GB/US/FR address-context tuples JSONL. Default the house-venue v3 tuples
+	 * (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`); DE and ES read OpenAddresses directly.
+	 */
+	subVenueTuples?: string
+	/**
+	 * `sub-venue`: share of emitted rows that are confound NEGATIVES. Default 0.3.
+	 */
+	negativeFraction?: number
 }
 
 /**