@mailwoman/corpus 8.6.0 → 9.1.0

package/src/shard-recipes/intersection.ts

@@ -30,21 +30,21 @@
  *   External inputs (`--edges-dir`, opts.edgesDir; both already on disk — do not re-download):
  *
  *   - <edges-dir>/tl_2023_{17031,34027,50023}_edges.shp (unzipped TIGER 2023 EDGES; default
- *       /tmp/tiger-edges)
- *   - /tmp/oa-cache/us__il__cook.zip (ZIP→city tails)
+ *       `$MAILWOMAN_DATA_ROOT/census/tiger2023-edges`, where `mailwoman situs interpolation` puts them)
+ *   - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
  */
 
-import { spawnSync } from "node:child_process"
 import { existsSync, readFileSync, writeFileSync } from "node:fs"
 
 import type { DuckDBConnection } from "@duckdb/node-api"
 import type { ComponentTag } from "@mailwoman/core/types"
-import { repoRootPath } from "@mailwoman/core/utils"
+import { dataRootPath, repoRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
+import { JSONSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow, LabeledRow } from "../types.ts"
-import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 interface County {
 	fips: string
@@ -60,7 +60,13 @@ const TRAIN_COUNTIES: readonly County[] = [
 const GOLDEN_COUNTIES: readonly County[] = [{ fips: "50023", state: "VT", regime: "rural" }]
 
 const EVAL_GOLD_PATH = repoRootPath("data", "eval", "external", "intersection-real.jsonl")
-const OA_COOK = { zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv" }
+/**
+ * Where `mailwoman situs interpolation` unpacks the per-county TIGER EDGES shapefiles — a Census download, so it sits
+ * beside the other Census vintages under `census/`. `--edges-dir` overrides it.
+ */
+const DEFAULT_EDGES_DIR = dataRootPath("census", "tiger2023-edges")
+
+const OA_COOK = { zip: dataRootPath("oa-cache", "us__il__cook.zip"), csv: "us/il/cook.csv" }
 
 /**
  * One real crossing extracted from a county's TIGER EDGES shapefile.
@@ -157,52 +163,22 @@ function weightedPick<T extends { w: number }>(items: readonly T[], random: () =
 }
 
 /**
- * Minimal RFC-4180-ish splitter (handles quoted fields) — same as the affix builder.
+ * Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
  */
-function splitCSV(line: string): string[] {
-	const out: string[] = []
-	let cur = ""
-	let inQ = false
-
-	for (let i = 0; i < line.length; i++) {
-		const c = line[i]
-
-		if (inQ) {
-			if (c === '"') {
-				if (line[i + 1] === '"') {
-					cur += '"'
-
-					i++
-				} else {
-					inQ = false
-				}
-			} else {
-				cur += c
-			}
-		} else if (c === '"') {
-			inQ = true
-		} else if (c === ",") {
-			out.push(cur)
-			cur = ""
-		} else {
-			cur += c
-		}
-	}
-
-	out.push(cur)
-
-	return out
-}
+const pairKey = (a: string, b: string): string => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u001F")
 
 /**
- * Order-insensitive crossing key, for eval-leakage exclusion + pair dedup.
+ * One row of the intersection eval gold — the shape this recipe reads to exclude eval crossings.
  */
-const pairKey = (a: string, b: string): string => [a.toLowerCase(), b.toLowerCase()].toSorted().join("\u001F")
+interface EvalGoldRow {
+	node: number | string
+	components: { intersection_a: string; intersection_b: string }
+}
 
 /**
  * Load the eval's crossings so neither train nor golden ever sees them.
  */
-function readEvalExclusions(): { nodes: Set<number>; pairs: Set<string> } {
+async function readEvalExclusions(): Promise<{ nodes: Set<number>; pairs: Set<string> }> {
 	const nodes = new Set<number>()
 	const pairs = new Set<string>()
 
@@ -212,9 +188,7 @@ function readEvalExclusions(): { nodes: Set<number>; pairs: Set<string> } {
 		return { nodes, pairs }
 	}
 
-	for (const line of readFileSync(EVAL_GOLD_PATH, "utf8").split("\n")) {
-		if (!line) continue
-		const row = JSON.parse(line)
+	for await (const row of JSONSpliterator.fromAsync<EvalGoldRow>(EVAL_GOLD_PATH)) {
 		nodes.add(Number(row.node))
 		pairs.add(pairKey(row.components.intersection_a, row.components.intersection_b))
 	}
@@ -276,29 +250,13 @@ async function extractCrossings(
 /**
  * ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
  */
-function buildZipCityMap(): Map<string, string> {
-	const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${OA_COOK.zip} — city tails disabled`)
-
-		return new Map()
-	}
-
-	const lines = r.stdout.toString("utf8").split(/\r?\n/)
-
-	if (lines.length < 2) return new Map()
-	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
-	const iCity = header.indexOf("city")
-	const iPost = header.indexOf("postcode")
+async function buildZipCityMap(): Promise<Map<string, string>> {
 	const counts = new Map<string, Map<string, number>>()
 
 	// zip → Map(city → n)
-	for (let li = 1; li < lines.length; li++) {
-		if (!lines[li]) continue
-		const cells = splitCSV(lines[li]!)
-		const city = (cells[iCity] ?? "").trim()
-		const zip = (cells[iPost] ?? "").trim()
+	for await (const row of readZippedCSVRecords(OA_COOK.zip, OA_COOK.csv)) {
+		const city = row.city ?? ""
+		const zip = row.postcode ?? ""
 
 		if (!city || !/^\d{5}$/.test(zip) || BAD_NAME.test(city)) continue
 		let byCity = counts.get(zip)
@@ -486,15 +444,20 @@ export const intersectionRecipe: ShardRecipe = {
 	name: "intersection",
 	description: "Real-pair intersection rows (US): TIGER 2023 EDGES crossings → audited intersection_a/b labels",
 	mode: "generate",
-	options: [{ flag: "--edges-dir <dir>", description: "Unzipped TIGER 2023 EDGES dir. Default /tmp/tiger-edges" }],
+	options: [
+		{
+			flag: "--edges-dir <dir>",
+			description: `Unzipped TIGER 2023 EDGES dir. Default ${DEFAULT_EDGES_DIR}`,
+		},
+	],
 	async run(opts, write) {
 		// Legacy build-intersection-shard.mjs seeded `mulberry32(opts.seed)`.
 		const random = makeMulberry32(opts.seed)
 		const count = opts.count ?? 40_000
 		const source = opts.sourceName ?? "synth-intersection"
-		const edgesDir = opts.edgesDir ?? "/tmp/tiger-edges"
+		const edgesDir = opts.edgesDir ?? DEFAULT_EDGES_DIR
 		const counties = opts.golden ? GOLDEN_COUNTIES : TRAIN_COUNTIES
-		const exclusions = readEvalExclusions()
+		const exclusions = await readEvalExclusions()
 
 		console.error(`  eval exclusions: ${exclusions.nodes.size} nodes, ${exclusions.pairs.size} pairs`)
 
@@ -546,7 +509,7 @@ export const intersectionRecipe: ShardRecipe = {
 			throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`)
 		}
 
-		const zipCity = opts.golden ? new Map<string, string>() : buildZipCityMap()
+		const zipCity = opts.golden ? new Map<string, string>() : await buildZipCityMap()
 
 		if (!opts.golden) {
 			console.error(`  zip→city map: ${zipCity.size} ZIPs (OA Cook)`)

package/src/shard-recipes/locale.ts

@@ -7,7 +7,7 @@
  *   OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
  *   {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
  *   rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
- *   source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
+ *   source CSV (a streamed zip member for cached zips, plain `createReadStream` for extracted CSVs) and
  *   reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
  *   draws `--count` rows from the pool with the passed `random`. Ported from
  *   scripts/build-locale-shard.mjs.
@@ -25,21 +25,25 @@
  *   country, so DE/FR emit streams are unchanged for a given seed.
  */
 
-import { spawn } from "node:child_process"
 import { createReadStream } from "node:fs"
 
 import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
+import { readZipEntry } from "@mailwoman/core/fs/zip"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import {
+	type LocaleBaseTuple,
+	type SynthesizedLocaleRow,
+	synthesizeLocaleRow,
+} from "@mailwoman/corpus/synthesizers/german"
+import { alignRow } from "@mailwoman/corpus/utils"
 import { CSVSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "../synthesize-german.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /**
- * One per-country OA source part: either a cached `zip` + `csv` member (streamed via `unzip -p`) or an extracted plain
- * `path` (streamed via `createReadStream`). Both carry the standard OA header
+ * One per-country OA source part: either a cached `zip` + `csv` member (streamed out of the archive) or an extracted
+ * plain `path` (streamed via `createReadStream`). Both carry the standard OA header
  * (LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID,HASH). An optional `region` fallback covers countries
  * whose REGION column is empty (DE — the Bundesland is implied by the per-state file).
  */
@@ -86,26 +90,26 @@ export interface LocaleCountrySource {
 }
 
 /**
- * Per-country OA sources + the source name used in the corpus. DE/FR still point at the legacy `/tmp/oa-cache` zips
- * (their historical build inputs — materialize them there to regenerate). ES/NL read the extracted countrywide CSVs and
- * IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is OA-conformed, so the old
- * raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is empty; the
- * international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad autónoma, IT =
- * regione, NL = province).
+ * Per-country OA sources + the source name used in the corpus. DE/FR read their historical build inputs, the cached
+ * zips under `$MAILWOMAN_DATA_ROOT/oa-cache` — materialize them there to regenerate. ES/NL read the extracted
+ * countrywide CSVs and IT the cached national zip under `$MAILWOMAN_DATA_ROOT` (#241; the fresh ES extract is
+ * OA-conformed, so the old raw-CNIG conform map is gone). DE carries a per-part `region` fallback (its REGION column is
+ * empty; the international-order tail needs it, #327). FR/NL/IT/ES REGION is populated per-row (ES = comunidad
+ * autónoma, IT = regione, NL = province).
  */
 const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
 	DE: {
 		source: "synth-german",
 		corpusVersion: "0.4.0",
 		parts: [
-			{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
-			{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
+			{ zip: dataRootPath("oa-cache", "de__berlin.zip"), csv: "de/berlin.csv", region: "Berlin" },
+			{ zip: dataRootPath("oa-cache", "de__sn__statewide.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
 		],
 	},
 	FR: {
 		source: "synth-fr",
 		corpusVersion: "0.4.0",
-		parts: [{ zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }],
+		parts: [{ zip: dataRootPath("oa-cache", "fr__countrywide.zip"), csv: "fr/countrywide.csv" }],
 	},
 	NL: {
 		source: "synth-nl",
@@ -144,7 +148,7 @@ const COUNTRY_SOURCES: Record<string, LocaleCountrySource> = {
 		parts: [{ path: dataRootPath("openaddresses", "extracted", "nz", "countrywide.csv"), districtAsLocality: true }],
 	},
 	GB: {
-		// HM Land Registry Price Paid Data tuples (25.67M rows; see Task 2's ppd ingest). PPD's DISTRICT is the
+		// HM Land Registry Price Paid Data tuples (25.67M rows out of the PPD ingest). PPD's DISTRICT is the
 		// postal town (locality) and CITY is the dependent locality — legitimately EMPTY on the majority of rows
 		// (most GB addresses have no dependent locality). `districtAsLocality` maps DISTRICT→locality and, when
 		// present, CITY→dependent_locality; the `readTuples` gate above only drops a row when BOTH are empty, so
@@ -223,7 +227,7 @@ interface ColumnIndex {
 
 /**
  * Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV row-by-row —
- * `unzip -p | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
+ * `readZipEntry | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
  * memory) — and keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO
  * global dedup (a 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the
  * region falls back to `part.region` when the row's REGION cell is empty (DE).
@@ -231,21 +235,11 @@ interface ColumnIndex {
  * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
  */
 export async function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]> {
-	let input: NodeJS.ReadableStream
-
-	if (part.path) {
-		// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
-		// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
-		input = createReadStream(part.path)
-	} else {
-		const child = spawn("unzip", ["-p", part.zip!, part.csv!])
-
-		child.on("error", (err) => {
-			console.error(`  WARN: unzip failed for ${part.zip}: ${err.message}`)
-		})
-
-		input = child.stdout!
-	}
+	// No `encoding` on the file path — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
+	// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
+	const input: NodeJS.ReadableStream | AsyncIterable<Uint8Array> = part.path
+		? createReadStream(part.path)
+		: readZipEntry(part.zip!, part.csv!)
 
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const reservoir: LocaleBaseTuple[] = []
@@ -256,9 +250,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
 
 	try {
 		// CSVSpliterator handles OA's quoted fields (embedded commas/newlines) and CRLF row terminators
-		// (spliterator ≥ 3.2.0); `header: false` yields the header row too, so we build the column index
-		// from it exactly as the prior hand-rolled split did. Verified byte-identical tuples vs that split
-		// on 10M+ real OA rows — the only raw-cell difference was a trailing CR on the discarded HASH column.
+		// (spliterator ≥ 3.2.0); `header: false` yields the header row too, so the column index is built
+		// from that first row rather than from a header option.
 		for await (const cells of CSVSpliterator.fromAsync<string[]>(input, {
 			mode: "array",
 			header: false,
@@ -315,9 +308,8 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
 			// dependent_locality. When DISTRICT is empty (~18% of NZ rows), fall back to CITY → locality with no
 			// sub-locality. GB PPD tuples flip which side is legitimately empty — on the MAJORITY of GB rows CITY
 			// (the dependent_locality) is empty and DISTRICT (the locality) is populated, so the gate below only
-			// drops a `districtAsLocality` row when BOTH are empty, not when CITY alone is (that used to silently
-			// drop most of the GB source — see the fixed `readTuples` gate below). See
-			// {@link LocalePart.districtAsLocality}.
+			// drops a `districtAsLocality` row when BOTH are empty, never when CITY alone is — gating on CITY
+			// alone silently discards most of the GB source. See {@link LocalePart.districtAsLocality}.
 			let locality: string | null
 			let dependent_locality: string | undefined
 

package/src/shard-recipes/no-fragment.ts

@@ -35,7 +35,7 @@
  *   list. Diacritic-KEEPING normalizer, matching the board (see the norm docstring).
  */
 
-import { readFileSync } from "node:fs"
+import { TextSpliterator } from "spliterator"
 
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
@@ -100,12 +100,15 @@ export const noFragmentRecipe: ShardRecipe = {
 			)
 		}
 
-		const excluded = new Set(
-			readFileSync(excludePath, "utf8")
-				.split("\n")
-				.map((line) => line.trim())
-				.filter((line) => line && !line.startsWith("#"))
-		)
+		const excluded = new Set<string>()
+
+		for await (const line of TextSpliterator.fromAsync(excludePath)) {
+			const trimmed = line.trim()
+
+			if (trimmed && !trimmed.startsWith("#")) {
+				excluded.add(trimmed)
+			}
+		}
 
 		if (!excluded.size) throw new Error(`no-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)
 

package/src/shard-recipes/no-street-led.ts

@@ -16,7 +16,7 @@
  *   3. pc-first   "«pc» «city», «st» «n»"   (the 7% floor — the anchor the others converge to)
  */
 
-import { readFileSync } from "node:fs"
+import { TextSpliterator } from "spliterator"
 
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
@@ -62,12 +62,15 @@ export const noStreetLedRecipe: ShardRecipe = {
 			)
 		}
 
-		const excluded = new Set(
-			readFileSync(excludePath, "utf8")
-				.split("\n")
-				.map((line) => line.trim())
-				.filter((line) => line && !line.startsWith("#"))
-		)
+		const excluded = new Set<string>()
+
+		for await (const line of TextSpliterator.fromAsync(excludePath)) {
+			const trimmed = line.trim()
+
+			if (trimmed && !trimmed.startsWith("#")) {
+				excluded.add(trimmed)
+			}
+		}
 
 		if (!excluded.size) throw new Error(`no-street-led: --exclude-surfaces "${excludePath}" listed no surfaces`)
 

package/src/shard-recipes/no-street.ts

@@ -10,7 +10,8 @@
  *   scripts/build-no-street-shard.mjs.
  */
 
-import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "../synthesize-no-street.ts"
+import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "@mailwoman/corpus/synthesizers/no-street"
+
 import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"