@mailwoman/corpus 9.0.0 → 9.1.0

package/src/name-prone-us-suffixes.ts

@@ -0,0 +1,12 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Re-export shim: the name-prone set moved to `@mailwoman/codex/us` (data in
+ * `codex/us/street-suffix.json`, 2026-08-10) so the golden relabel flags, the #1569 shard
+ * recipe, AND the Python relabel pass (via the `gazetteer affix-relabel` v2 lexicon) all read
+ * one record. Import from `@mailwoman/codex/us` in new code.
+ */
+
+export { NAME_PRONE_US_SUFFIXES } from "@mailwoman/codex/us"

package/src/runner.ts

@@ -31,8 +31,13 @@ import { createWriteStream, type WriteStream } from "node:fs"
 import { mkdir, writeFile } from "node:fs/promises"
 import { dirname, join } from "node:path"
 
-import { canonicalDedupKey, streamingSha256, type AdapterRegistry, type StreamingHasher } from "./adapter.ts"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "./types.ts"
+import {
+	canonicalDedupKey,
+	streamingSha256,
+	type AdapterRegistry,
+	type StreamingHasher,
+} from "@mailwoman/corpus/adapters/utils"
+import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "@mailwoman/corpus/types"
 
 /**
  * Snapshot of the runner's state, emitted on every progress tick.

package/src/shard-recipes/anchor-absorption.ts

@@ -16,10 +16,10 @@ import { readFileSync } from "node:fs"
 
 import { parseJSONStrict } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { synthesizeAnchorAbsorptionRow } from "@mailwoman/corpus/synthesizers/anchor-absorption"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.ts"
 import { makeLcg, type ShardRecipe } from "./scaffold.ts"
 
 /**

package/src/shard-recipes/boundary-stress.ts

@@ -12,8 +12,12 @@
  *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
  */
 
-import { alignRow } from "../align.ts"
-import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.ts"
+import {
+	type BoundaryStressTemplate,
+	synthesizeBoundaryStressRow,
+} from "@mailwoman/corpus/synthesizers/boundary-stress"
+import { alignRow } from "@mailwoman/corpus/utils"
+
 import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
 /**

package/src/shard-recipes/country-balanced.ts

@@ -24,16 +24,14 @@
  *   legacy script used.
  */
 
-import { spawnSync } from "node:child_process"
-
 import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 // v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
 // ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
@@ -154,26 +152,15 @@ interface CountryTuple {
 	order: string
 }
 
-function readTuples(source: CountrySource, limit: number): CountryTuple[] {
-	// countrywide extracts (FR/IT/NL) are GB-scale — cap the bytes with `head` (read ~8 lines per wanted
-	// tuple to survive dedup/skips) so the whole extract never has to be held in memory.
-	const maxLines = Math.max(limit * 8, 20_000) + 1
-
-	const r = spawnSync("bash", ["-c", `unzip -p "${source.zip}" "${source.csv}" | head -n ${maxLines}`], {
-		maxBuffer: 1024 * 1024 * 1024,
-		encoding: "buffer",
-	})
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
-
-		return []
-	}
-
+/**
+ * The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
+ * closes the reader, which releases the archive without inflating the rest of the member.
+ */
+async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
 	const tuples: CountryTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
 		if (tuples.length >= limit) break
 
 		const street = row.street ?? "",
@@ -449,7 +436,7 @@ export const countryBalancedRecipe: ShardRecipe = {
 		const pool: CountryTuple[] = []
 
 		for (const s of sources) {
-			const t = readTuples(s, perSource)
+			const t = await readTuples(s, perSource)
 
 			console.error(`  ${s.csv} (${s.iso2}): ${t.length} tuples`)
 

package/src/shard-recipes/fr-admin-split.ts

@@ -32,11 +32,11 @@
 
 import { departementForCodePostal } from "@mailwoman/codex/fr"
 import type { ComponentTag } from "@mailwoman/core/types"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 import { CSVSpliterator, Delimiters } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler

package/src/shard-recipes/fr-lieudit.ts

@@ -31,11 +31,11 @@ import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
 import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
 import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /**

package/src/shard-recipes/fr-order.ts

@@ -30,15 +30,13 @@
  *   inlined the variant logic and never called it).
  */
 
-import { spawnSync } from "node:child_process"
-
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
@@ -72,28 +70,15 @@ interface FrTuple {
 }
 
 /**
- * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` so the whole file
- * never has to be held in memory. Only keeps rows with a house_number (the shard's core signal) and a postcode
- * (required for reversed-order rendering to be meaningful).
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
+ * distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
+ * shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
  */
-function readTuples(limit: number): FrTuple[] {
-	const maxLines = Math.max(limit * 8, 40_000) + 1
-
-	const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
-		maxBuffer: 1024 * 1024 * 1024,
-		encoding: "buffer",
-	})
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`)
-
-		return []
-	}
-
+async function readTuples(limit: number): Promise<FrTuple[]> {
 	const tuples: FrTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
 		if (tuples.length >= limit) break
 		const street = row.street ?? ""
 		const locality = row.city ?? ""
@@ -161,7 +146,7 @@ export const frOrderRecipe: ShardRecipe = {
 
 		// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
 		const poolLimit = Math.max(count * 8, 40_000)
-		const pool = readTuples(poolLimit)
+		const pool = await readTuples(poolLimit)
 
 		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
 

package/src/shard-recipes/german.ts

@@ -15,14 +15,12 @@
  *   docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
  */
 
-import { spawnSync } from "node:child_process"
-
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import { synthesizeGermanRow, type LocaleBaseTuple } from "@mailwoman/corpus/synthesizers/german"
+import { alignRow } from "@mailwoman/corpus/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /**
  * A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
@@ -44,21 +42,13 @@ const SOURCES: GermanSource[] = [
 ]
 
 /**
- * Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
+ * Stream real German tuples out of a cached OA zip.
  */
-function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
-	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
-
-		return []
-	}
-
+async function readGermanTuples(source: GermanSource): Promise<LocaleBaseTuple[]> {
 	const tuples: LocaleBaseTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
 		const street = row.street ?? ""
 		const locality = row.city ?? ""
 
@@ -102,7 +92,7 @@ export const germanRecipe: ShardRecipe = {
 		const pool: LocaleBaseTuple[] = []
 
 		for (const s of SOURCES) {
-			const t = readGermanTuples(s)
+			const t = await readGermanTuples(s)
 
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 

package/src/shard-recipes/house-venue.ts

@@ -10,7 +10,8 @@
  *   scripts/build-house-venue-shard.mjs.
  */
 
-import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "../synthesize-house-venue.ts"
+import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "@mailwoman/corpus/synthesizers/house-venue"
+
 import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"

package/src/shard-recipes/index.ts

@@ -29,7 +29,7 @@ import { poBoxCedexRecipe } from "./po-box-cedex.ts"
 import { poBoxRecipe } from "./po-box.ts"
 import type { ShardRecipe } from "./scaffold.ts"
 import { siBareVillageRecipe } from "./si-bare-village.ts"
-import { streetAffixRecipe } from "./street-affix.ts"
+import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.ts"
 import { streetBareRecipe } from "./street-bare.ts"
 import { streetRecipe } from "./street.ts"
 import { subVenueRecipe } from "./sub-venue.ts"
@@ -44,6 +44,7 @@ const RECIPES: readonly ShardRecipe[] = [
 	streetRecipe,
 	streetBareRecipe,
 	streetAffixRecipe,
+	suffixBoundaryRecipe,
 	noStreetRecipe,
 	houseVenueRecipe,
 	poBoxRecipe,

package/src/shard-recipes/intersection.ts

@@ -34,18 +34,17 @@
  *   - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
  */
 
-import { spawnSync } from "node:child_process"
 import { existsSync, readFileSync, writeFileSync } from "node:fs"
 
 import type { DuckDBConnection } from "@duckdb/node-api"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath, repoRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import type { CanonicalRow, LabeledRow } from "@mailwoman/corpus/types"
+import { alignRow } from "@mailwoman/corpus/utils"
 import { JSONSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow, LabeledRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 interface County {
 	fips: string
@@ -251,19 +250,11 @@ async function extractCrossings(
 /**
  * ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
  */
-function buildZipCityMap(): Map<string, string> {
-	const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${OA_COOK.zip} — city tails disabled`)
-
-		return new Map()
-	}
-
+async function buildZipCityMap(): Promise<Map<string, string>> {
 	const counts = new Map<string, Map<string, number>>()
 
 	// zip → Map(city → n)
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(OA_COOK.zip, OA_COOK.csv)) {
 		const city = row.city ?? ""
 		const zip = row.postcode ?? ""
 
@@ -518,7 +509,7 @@ export const intersectionRecipe: ShardRecipe = {
 			throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`)
 		}
 
-		const zipCity = opts.golden ? new Map<string, string>() : buildZipCityMap()
+		const zipCity = opts.golden ? new Map<string, string>() : await buildZipCityMap()
 
 		if (!opts.golden) {
 			console.error(`  zip→city map: ${zipCity.size} ZIPs (OA Cook)`)

package/src/shard-recipes/locale.ts

@@ -7,7 +7,7 @@
  *   OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
  *   {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
  *   rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
- *   source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
+ *   source CSV (a streamed zip member for cached zips, plain `createReadStream` for extracted CSVs) and
  *   reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
  *   draws `--count` rows from the pool with the passed `random`. Ported from
  *   scripts/build-locale-shard.mjs.
@@ -25,21 +25,25 @@
  *   country, so DE/FR emit streams are unchanged for a given seed.
  */
 
-import { spawn } from "node:child_process"
 import { createReadStream } from "node:fs"
 
 import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
+import { readZipEntry } from "@mailwoman/core/fs/zip"
 import { dataRootPath } from "@mailwoman/core/utils"
+import { stableSourceID } from "@mailwoman/corpus/adapters/utils"
+import {
+	type LocaleBaseTuple,
+	type SynthesizedLocaleRow,
+	synthesizeLocaleRow,
+} from "@mailwoman/corpus/synthesizers/german"
+import { alignRow } from "@mailwoman/corpus/utils"
 import { CSVSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "../synthesize-german.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /**
- * One per-country OA source part: either a cached `zip` + `csv` member (streamed via `unzip -p`) or an extracted plain
- * `path` (streamed via `createReadStream`). Both carry the standard OA header
+ * One per-country OA source part: either a cached `zip` + `csv` member (streamed out of the archive) or an extracted
+ * plain `path` (streamed via `createReadStream`). Both carry the standard OA header
  * (LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID,HASH). An optional `region` fallback covers countries
  * whose REGION column is empty (DE — the Bundesland is implied by the per-state file).
  */
@@ -223,7 +227,7 @@ interface ColumnIndex {
 
 /**
  * Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV row-by-row —
- * `unzip -p | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
+ * `readZipEntry | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
  * memory) — and keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO
  * global dedup (a 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the
  * region falls back to `part.region` when the row's REGION cell is empty (DE).
@@ -231,21 +235,11 @@ interface ColumnIndex {
  * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
  */
 export async function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]> {
-	let input: NodeJS.ReadableStream
-
-	if (part.path) {
-		// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
-		// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
-		input = createReadStream(part.path)
-	} else {
-		const child = spawn("unzip", ["-p", part.zip!, part.csv!])
-
-		child.on("error", (err) => {
-			console.error(`  WARN: unzip failed for ${part.zip}: ${err.message}`)
-		})
-
-		input = child.stdout!
-	}
+	// No `encoding` on the file path — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
+	// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
+	const input: NodeJS.ReadableStream | AsyncIterable<Uint8Array> = part.path
+		? createReadStream(part.path)
+		: readZipEntry(part.zip!, part.csv!)
 
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const reservoir: LocaleBaseTuple[] = []

package/src/shard-recipes/no-street.ts

@@ -10,7 +10,8 @@
  *   scripts/build-no-street-shard.mjs.
  */
 
-import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "../synthesize-no-street.ts"
+import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "@mailwoman/corpus/synthesizers/no-street"
+
 import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"