@mailwoman/corpus 8.0.0 → 8.2.0

package/src/shard-recipes/fr-lieudit.ts

@@ -38,9 +38,14 @@ import { alignRow } from "../align.ts"
 import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
-const DEFAULT_LICENSE = "Licence Ouverte 2.0" // matches the `ban` adapter's Tier-B election for BAN data
+/**
+ * Matches the `ban` adapter's Tier-B election for BAN data.
+ */
+const DEFAULT_LICENSE = "Licence Ouverte 2.0"
 
-/** One BAN row surviving the lieu-dit filter — the minimal tuple the pool holds. */
+/**
+ * One BAN row surviving the lieu-dit filter — the minimal tuple the pool holds.
+ */
 interface LieuDitTuple {
 	numero: string
 	rep: string | null
@@ -60,7 +65,7 @@ interface LieuDitTuple {
 function departementFiles(banDir: string): string[] {
 	const byDept = new Map<string, string>()
 
-	for (const name of readdirSync(banDir).sort()) {
+	for (const name of readdirSync(banDir).toSorted()) {
 		const m = /^adresses-(.+?)\.csv(\.gz)?$/.exec(name)
 
 		if (!m) continue
@@ -76,14 +81,16 @@ function departementFiles(banDir: string): string[] {
 		}
 	}
 
-	return [...byDept.keys()].sort().map((dept) => byDept.get(dept)!)
+	return [...byDept.keys()].toSorted().map((dept) => byDept.get(dept)!)
 }
 
-/** Stream every département file, keeping only rows with a clean `lieuDit` (junk/dup filtering lives in `ban/sdk`). */
+/**
+ * Stream every département file, keeping only rows with a clean `lieuDit` (junk/dup filtering lives in `ban/sdk`).
+ */
 async function readLieuDitPool(banDir: string): Promise<LieuDitTuple[]> {
 	const files = departementFiles(banDir)
 
-	if (files.length === 0) {
+	if (!files.length) {
 		throw new Error(
 			`No BAN adresses-<dept>.csv files found in ${banDir} — fetch BAN first (\`mailwoman corpus fetch ban\`).`
 		)
@@ -108,10 +115,13 @@ async function readLieuDitPool(banDir: string): Promise<LieuDitTuple[]> {
 				locality: rec.city,
 				dependentLocality: rec.lieuDit,
 			})
+
 			deptCount++
 		}
+
 		console.error(`  ${path}: ${deptCount.toLocaleString()} clean lieu-dit rows`)
 	}
+
 	console.error(
 		`  scanned ${scanned.toLocaleString()} BAN rows across ${files.length} départements → pool=${pool.length.toLocaleString()}`
 	)
@@ -119,7 +129,9 @@ async function readLieuDitPool(banDir: string): Promise<LieuDitTuple[]> {
 	return pool
 }
 
-/** `house_number` = `numero` + folded `rep` ("10 bis"), matching the `ban` adapter's own composition. */
+/**
+ * `house_number` = `numero` + folded `rep` ("10 bis"), matching the `ban` adapter's own composition.
+ */
 function composeHouseNumber(numero: string, rep: string | null): string {
 	return rep ? `${numero} ${rep}` : numero
 }
@@ -142,6 +154,7 @@ function composeRaw(
 	if (streetLine) {
 		lines.push(streetLine)
 	}
+
 	lines.push(dependentLocality)
 	const cityLine = [postcode, locality].filter(Boolean).join(" ").trim()
 
@@ -152,7 +165,9 @@ function composeRaw(
 	return lines.join("\n")
 }
 
-/** Fisher-Yates shuffle, in place, with the recipe's seeded PRNG — reproducible sampling without replacement. */
+/**
+ * Fisher-Yates shuffle, in place, with the recipe's seeded PRNG — reproducible sampling without replacement.
+ */
 function shuffleInPlace<T>(arr: T[], random: () => number): void {
 	for (let i = arr.length - 1; i > 0; i--) {
 		const j = Math.floor(random() * (i + 1))
@@ -160,6 +175,10 @@ function shuffleInPlace<T>(arr: T[], random: () => number): void {
 	}
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const frLieuditRecipe: ShardRecipe = {
 	name: "fr-lieudit",
 	description: "FR lieu-dit rows: BAN nom_ld → dependent_locality (commune → locality), lieu-dit on its own line",
@@ -187,7 +206,7 @@ export const frLieuditRecipe: ShardRecipe = {
 
 		const pool = await readLieuDitPool(banDir)
 
-		if (pool.length === 0) {
+		if (!pool.length) {
 			throw new Error(`No clean lieu-dit rows found under ${banDir} — see ban/sdk/extract.ts's cleanLieuDit filter.`)
 		}
 
@@ -224,6 +243,7 @@ export const frLieuditRecipe: ShardRecipe = {
 
 			if (!raw) {
 				skipped++
+
 				continue
 			}
 
@@ -237,6 +257,7 @@ export const frLieuditRecipe: ShardRecipe = {
 				const form = forms[Math.floor(random() * forms.length)]!
 				raw = `${raw}, ${form}`
 				components.country = form
+
 				countryAppended++
 			}
 
@@ -247,6 +268,7 @@ export const frLieuditRecipe: ShardRecipe = {
 				locality: t.locality,
 				postcode: t.postcode ?? undefined,
 			})
+
 			const canonical: CanonicalRow = {
 				raw,
 				components,
@@ -257,13 +279,17 @@ export const frLieuditRecipe: ShardRecipe = {
 				corpus_version: "",
 				license: DEFAULT_LICENSE,
 			}
+
 			const aligned = alignRow(canonical)
 
 			if (aligned.kind !== "labeled" || !aligned.row) {
 				skipped++
+
 				continue
 			}
+
 			write(JSON.stringify({ ...aligned.row, synth_method: source, synth_base_id: null }) + "\n")
+
 			emitted++
 		}
 

package/src/shard-recipes/fr-order.ts

@@ -39,16 +39,30 @@ import { alignRow } from "../align.ts"
 import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
+/* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
+   cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
+   output distribution, and reading the cascade top-to-bottom is how you see it. Naming each cutoff
+   would hide the distribution behind a wall of identifiers. Genuine thresholds in these files are
+   extracted as named constants above. */
+
 const SOURCE = { zip: "/tmp/oa-cache/fr__countrywide.zip", csv: "fr/countrywide.csv" }
 
-// Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
+/**
+ * Ordinal suffixes used in French house numbers (BAN corpus), to cover the "8 bis" sub-mode.
+ */
 const ORDINAL_SUFFIXES: readonly string[] = ["bis", "ter", "quater"]
-// Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
+/**
+ * Probability that a row gets an ordinal suffix injected (matches the golden's ~10-15% rate).
+ */
 const ORDINAL_PROB = 0.12
-// Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
+/**
+ * Probability that a locality renders ALL-CAPS (another sub-mode: "SAINTE-LIVRADE-SUR-LOT").
+ */
 const ALLCAPS_PROB = 0.1
 
-/** A real FR tuple read out of the cached OA zip. */
+/**
+ * A real FR tuple read out of the cached OA zip.
+ */
 interface FrTuple {
 	house_number: string
 	street: string
@@ -56,7 +70,9 @@ interface FrTuple {
 	postcode: string
 }
 
-/** Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping). */
+/**
+ * Minimal RFC-4180-ish splitter (handles quoted fields with doubled-quote escaping).
+ */
 function splitCSV(line: string): string[] {
 	const out: string[] = []
 	let cur = ""
@@ -69,6 +85,7 @@ function splitCSV(line: string): string[] {
 			if (c === '"') {
 				if (line[i + 1] === '"') {
 					cur += '"'
+
 					i++
 				} else {
 					inQ = false
@@ -85,6 +102,7 @@ function splitCSV(line: string): string[] {
 			cur += c
 		}
 	}
+
 	out.push(cur)
 
 	return out
@@ -96,7 +114,8 @@ function splitCSV(line: string): string[] {
  * reversed-order rendering to be meaningful).
  */
 function readTuples(limit: number): FrTuple[] {
-	const maxLines = Math.max(limit * 8, 40000) + 1
+	const maxLines = Math.max(limit * 8, 40_000) + 1
+
 	const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
 		maxBuffer: 1024 * 1024 * 1024,
 		encoding: "buffer",
@@ -107,15 +126,18 @@ function readTuples(limit: number): FrTuple[] {
 
 		return []
 	}
+
 	const lines = r.stdout.toString("utf8").split(/\r?\n/)
 
 	if (lines.length < 2) return []
 	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
 	const idx = (name: string): number => header.indexOf(name)
+
 	const iNum = idx("number"),
 		iStreet = idx("street"),
 		iCity = idx("city"),
 		iPost = idx("postcode")
+
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: FrTuple[] = []
 	const seen = new Set<string>()
@@ -152,7 +174,9 @@ function maybeAddOrdinal(random: () => number, house_number: string): string {
 	return `${house_number} ${random() < 0.5 ? suffix : suffix.toUpperCase()}`
 }
 
-/** Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes". */
+/**
+ * Render a tuple in CANONICAL French order: "9 Rue de la Promenade, 01200 Villes".
+ */
 function renderCanonical(
 	hn: string,
 	street: string,
@@ -164,6 +188,10 @@ function renderCanonical(
 	return { raw, components: { house_number: hn, street, postcode, locality } }
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const frOrderRecipe: ShardRecipe = {
 	name: "fr-order",
 	description: "French reversed-order rows (#560): real OA FR tuples rendered canonical + 4 postcode-first variants",
@@ -182,11 +210,12 @@ export const frOrderRecipe: ShardRecipe = {
 		const reversedFraction = opts.reversedFraction ?? 0.5
 
 		// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
-		const poolLimit = Math.max(count * 8, 40000)
+		const poolLimit = Math.max(count * 8, 40_000)
 		const pool = readTuples(poolLimit)
+
 		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
 
-		if (pool.length === 0) {
+		if (!pool.length) {
 			throw new Error("No FR tuples found — is /tmp/oa-cache/fr__countrywide.zip present?")
 		}
 
@@ -223,6 +252,7 @@ export const frOrderRecipe: ShardRecipe = {
 					// Variant D: postcode, HN+street, city (reversed top-to-bottom)
 					raw = `${postcode}, ${house_number} ${street}, ${locality}`
 				}
+
 				rendered = { raw, components: { house_number, street, postcode, locality } }
 			} else {
 				rendered = renderCanonical(house_number, street, postcode, locality)
@@ -235,13 +265,16 @@ export const frOrderRecipe: ShardRecipe = {
 
 			if (!componentValues.every((v) => raw.includes(v))) {
 				skipped++
+
 				continue
 			}
 
 			// --golden: emit per-locale-f1 eval rows ({raw, components, country:"FR"}).
 			if (opts.golden) {
 				write(JSON.stringify({ raw, components, country: "FR" }) + "\n")
+
 				emitted++
+
 				continue
 			}
 
@@ -251,6 +284,7 @@ export const frOrderRecipe: ShardRecipe = {
 				locality: components.locality,
 				postcode: components.postcode,
 			})
+
 			const canonical: CanonicalRow = {
 				raw,
 				components,
@@ -261,12 +295,15 @@ export const frOrderRecipe: ShardRecipe = {
 				corpus_version: "0.5.0",
 				license: "OpenAddresses FR countrywide tuples, rendered canonical + reversed-order — see ingest SOURCE",
 			}
+
 			const aligned = alignRow(canonical)
 
 			if (aligned.kind !== "labeled" || !aligned.row) {
 				skipped++
+
 				continue
 			}
+
 			write(
 				JSON.stringify({
 					...aligned.row,
@@ -275,6 +312,7 @@ export const frOrderRecipe: ShardRecipe = {
 					synth_base_id: null,
 				}) + "\n"
 			)
+
 			emitted++
 		}
 

package/src/shard-recipes/german.ts

@@ -31,15 +31,19 @@ interface GermanSource {
 	region: string
 }
 
-// `region` is the Bundesland the source covers. OA's REGION column is empty for DE, but the region is
-// implied by the per-state file — the international order needs it for the "City, Region Postcode" tail
-// (v0.9.3 / #327). berlin.csv → Berlin (a city-state, region==locality); sn/statewide → Sachsen.
+/**
+ * `region` is the Bundesland the source covers. OA's REGION column is empty for DE, but the region is implied by the
+ * per-state file — the international order needs it for the "City, Region Postcode" tail (v0.9.3 / #327). berlin.csv →
+ * Berlin (a city-state, region==locality); sn/statewide → Sachsen.
+ */
 const SOURCES: GermanSource[] = [
 	{ zip: "/tmp/oa-cache/de__berlin.zip", csv: "de/berlin.csv", region: "Berlin" },
 	{ zip: "/tmp/oa-cache/de__sn__statewide.zip", csv: "de/sn/statewide.csv", region: "Sachsen" },
 ]
 
-/** Minimal RFC-4180-ish splitter (handles quoted fields). */
+/**
+ * Minimal RFC-4180-ish splitter (handles quoted fields).
+ */
 function splitCSV(line: string): string[] {
 	const out: string[] = []
 	let cur = ""
@@ -52,6 +56,7 @@ function splitCSV(line: string): string[] {
 			if (c === '"') {
 				if (line[i + 1] === '"') {
 					cur += '"'
+
 					i++
 				} else {
 					inQ = false
@@ -68,12 +73,15 @@ function splitCSV(line: string): string[] {
 			cur += c
 		}
 	}
+
 	out.push(cur)
 
 	return out
 }
 
-/** Stream real German tuples out of a cached OA zip (buffered `unzip -p`). */
+/**
+ * Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
+ */
 function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
 	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
 
@@ -82,16 +90,19 @@ function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
 
 		return []
 	}
+
 	const lines = r.stdout.toString("utf8").split(/\r?\n/)
 
 	if (lines.length < 2) return []
 	const header = splitCSV(lines[0]!).map((h) => h.trim().toLowerCase())
 	const idx = (name: string): number => header.indexOf(name)
+
 	const iNum = idx("number"),
 		iStreet = idx("street"),
 		iCity = idx("city"),
 		iRegion = idx("region"),
 		iPost = idx("postcode")
+
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const tuples: LocaleBaseTuple[] = []
 	const seen = new Set<string>()
@@ -119,6 +130,10 @@ function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
 	return tuples
 }
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const germanRecipe: ShardRecipe = {
 	name: "german",
 	description: "German coverage rows from real OA tuples (Berlin/Saxony), both orders → synthesizeGermanRow",
@@ -133,6 +148,7 @@ export const germanRecipe: ShardRecipe = {
 		if (!(intlFraction >= 0 && intlFraction <= 1)) {
 			throw new Error(`--intl-fraction must be in [0, 1], got ${intlFraction}`)
 		}
+
 		const count = opts.count ?? 4000
 
 		// Pool real tuples from every German source, then sample `count` rows from it.
@@ -140,6 +156,7 @@ export const germanRecipe: ShardRecipe = {
 
 		for (const s of SOURCES) {
 			const t = readGermanTuples(s)
+
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 
 			for (const x of t) {
@@ -147,7 +164,7 @@ export const germanRecipe: ShardRecipe = {
 			} // NOT pool.push(...t) — spreading ~840K args overflows the stack
 		}
 
-		if (pool.length === 0) {
+		if (!pool.length) {
 			throw new Error("No German tuples found — are the cached zips present in /tmp/oa-cache?")
 		}
 
@@ -165,6 +182,7 @@ export const germanRecipe: ShardRecipe = {
 
 			if (!synth) {
 				skipped++
+
 				continue
 			}
 
@@ -172,15 +190,19 @@ export const germanRecipe: ShardRecipe = {
 			// rides along so the eval can stratify native vs international.
 			if (opts.golden) {
 				write(JSON.stringify({ raw: synth.raw, components: synth.components, country: "DE", order }) + "\n")
+
 				emitted++
+
 				continue
 			}
+
 			const sourceID = stableSourceID(source, {
 				street: synth.components.street,
 				house_number: synth.components.house_number,
 				locality: synth.components.locality,
 				postcode: synth.components.postcode,
 			})
+
 			const canonical = {
 				raw: synth.raw,
 				components: synth.components,
@@ -191,13 +213,17 @@ export const germanRecipe: ShardRecipe = {
 				corpus_version: "0.4.0",
 				license: `OpenAddresses DE (Berlin/Saxony) tuples, rendered ${order}-order — see ingest SOURCES`,
 			}
+
 			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
 			if (aligned.kind !== "labeled" || !aligned.row) {
 				skipped++
+
 				continue
 			}
+
 			write(JSON.stringify({ ...aligned.row, synth_method: "german", synth_order: order, synth_base_id: null }) + "\n")
+
 			emitted++
 		}
 

package/src/shard-recipes/house-venue.ts

@@ -15,6 +15,10 @@ import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } f
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
 
+/**
+ * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
 export const houseVenueRecipe: ShardRecipe = {
 	name: "house-venue",
 	description: "House_number+venue co-occurrence rows: tuples → synthesizeHouseVenueRow → aligned LabeledRow",
@@ -33,6 +37,7 @@ export const houseVenueRecipe: ShardRecipe = {
 
 			if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
 				skipped++
+
 				continue
 			}
 
@@ -41,8 +46,10 @@ export const houseVenueRecipe: ShardRecipe = {
 
 				if (!synth) {
 					skipped++
+
 					continue
 				}
+
 				const ok = alignAndWrite(
 					write,
 					{

package/src/shard-recipes/index.ts

@@ -36,7 +36,9 @@ import { unitRecipe } from "./unit.ts"
 
 export * from "./scaffold.ts"
 
-/** Every registered recipe, in display order. */
+/**
+ * Every registered recipe, in display order.
+ */
 const RECIPES: readonly ShardRecipe[] = [
 	streetRecipe,
 	streetBareRecipe,
@@ -64,15 +66,21 @@ const RECIPES: readonly ShardRecipe[] = [
 	anchorAbsorptionRecipe,
 ]
 
-/** Recipe name → recipe. */
+/**
+ * Recipe name → recipe.
+ */
 export const SHARD_RECIPES: ReadonlyMap<string, ShardRecipe> = new Map(RECIPES.map((r) => [r.name, r]))
 
-/** Look up a recipe by its `<recipe>` name. */
+/**
+ * Look up a recipe by its `<recipe>` name.
+ */
 export function getShardRecipe(name: string): ShardRecipe | undefined {
 	return SHARD_RECIPES.get(name)
 }
 
-/** All recipes (for `--list` / help). */
+/**
+ * All recipes (for `--list` / help).
+ */
 export function listShardRecipes(): readonly ShardRecipe[] {
 	return RECIPES
 }