@mailwoman/corpus 9.0.0 → 9.2.0

package/src/shard-recipes/fr-bare-street.ts

@@ -20,9 +20,25 @@
  *   run including this shard.
  */
 
+import { FR_VOIE_TYPES } from "@mailwoman/codex/fr"
+
 import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
 import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
 
+/**
+ * Canonical voie type (lowercase, accent-kept) → its most common written abbreviation, from the codex table's first
+ * entry. Types with no attested abbreviation stay canonical in the abbreviated form.
+ */
+const FR_VOIE_ABBREV: Record<string, string> = Object.fromEntries(
+	Object.entries(FR_VOIE_TYPES).flatMap(([canonical, abbrevs]) => (abbrevs[0] ? [[canonical, abbrevs[0]]] : []))
+)
+
+/**
+ * The order-cycle slot for the BARE-STREET-ONLY form (`«voie» «name»`, no number, no locality) — the absence
+ * counterweight to the locality-terminated comma-free forms (see the cycle comment).
+ */
+const BARE_STREET_ONLY_FORM = 3
+
 /**
  * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
  * and `description` below for the surface form it generates.
@@ -30,7 +46,7 @@ import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourc
 export const frBareStreetRecipe: ShardRecipe = {
 	name: "fr-bare-street",
 	description:
-		"FR bare comma-form street+city, NO postcode (#251): '<n> Rue <name>, <City>' — the postcode-anchoring lever",
+		"FR bare street+city, NO postcode (#251): comma / comma-free / abbreviated-voie surfaces over one label set",
 	mode: "tuples",
 	async run(opts, write) {
 		// Seeded for parity with the other recipes; unused beyond reproducibility (the tuples drive the content).
@@ -53,23 +69,82 @@ export const frBareStreetRecipe: ShardRecipe = {
 
 			const { prefix, street } = decomposeFrStreet(fullStreet)
 
-			// The failing class is the prefix-led FR street; a no-prefix nom_voie ("La Ville Mois") isn't it.
+			// A no-prefix nom_voie ("La Ville Mois") is not the prefix-led class the numbered forms
+			// exercise — but as a BARE surface it is exactly the non-voie-led counterweight the v4.5.1
+			// probe showed missing ('Savile Row'-shaped spans still fell to the trailing-locality
+			// prior; the voie-led bare form protected only voie-led spans). Alternate rows emit the
+			// whole span as a bare street; the rest skip as before.
 			if (!prefix || !street) {
+				if (read % 2 === 0 && fullStreet.split(" ").length >= 2) {
+					const bare = {
+						raw: fullStreet,
+						components: { street: fullStreet } as Record<string, string>,
+						country: "FR",
+						locale: "fr-FR",
+						source: "synth-fr-bare-street",
+						source_id: shardSourceID("synth-fr-bare-street", {
+							street: fullStreet,
+							f: "bare-nonvoie",
+							v: String(read),
+						}),
+						corpus_version: "0.9.4",
+						license:
+							"Synthetic — fr-bare-street; (street, number, city) from BAN (Base Adresse Nationale, Licence Ouverte)",
+					}
+
+					if (alignAndWrite(write, bare, "fr-bare-street")) {
+						emitted++
+					} else {
+						skipped++
+					}
+
+					continue
+				}
+
 				skipped++
 
 				continue
 			}
 
-			const components: Record<string, string> = {
-				house_number: number,
-				street_prefix: prefix,
-				street,
-				locality,
+			// Four surfaces over the same tuple, cycled deterministically. The comma form was the
+			// original lever; the COMMA-FREE form is the colloquial register users actually type
+			// ('12 rue de Rome Paris' — the street↔locality boundary with NO delimiter, the fr-fr
+			// panel's named loss); the ABBREVIATED form is the typeahead register the geocoder-tester
+			// FR slice attests at scale; and the BARE-STREET-ONLY form is the absence counterweight —
+			// without it, every delimiter-free surface in the mix ENDS in a locality, the model learns
+			// "trailing span = locality" as categorical, and bare street names across locales flip to
+			// locality wholesale (the v4.5.0 no-promote's measured erosion: 'Calle de Alcalá',
+			// 'Madison Square West', and COMER's fork all fell to that prior). Tags are identical
+			// where present; each component VALUE is the span as written (BIO alignment binds value
+			// to surface); the bare form carries NO number and NO locality because the surface has
+			// neither.
+			const form = read % 4
+			const prefixSurface = form >= 2 ? (FR_VOIE_ABBREV[prefix.toLowerCase()] ?? prefix) : prefix
+
+			const components: Record<string, string> =
+				form === BARE_STREET_ONLY_FORM
+					? { street_prefix: prefix, street }
+					: { house_number: number, street_prefix: prefixSurface, street, locality }
+
+			// When the tuple carries a WOF-attested neighbourhood, the comma slot renders the
+			// THREE-slot middle surface — the dependent-locality counterweight (the v4.5.1 erosion's
+			// untouched half: the two-slot comma-free endings squeezed the middle tag out).
+			const hood = String(t.neighbourhood ?? "").trim()
+
+			if (form === 0 && hood) {
+				components.dependent_locality = hood
 			}
 
-			// BARE comma form, number-before (FR's dominant order), NO postcode — the whole point.
-			const raw = `${number} ${prefix} ${street}, ${locality}`
-			const source_id = shardSourceID("synth-fr-bare-street", { ...components, v: String(read) })
+			const raw =
+				form === 0
+					? hood
+						? `${number} ${prefix} ${street}, ${hood}, ${locality}`
+						: `${number} ${prefix} ${street}, ${locality}`
+					: form === BARE_STREET_ONLY_FORM
+						? `${prefix} ${street}`
+						: `${number} ${prefixSurface} ${street} ${locality}`
+
+			const source_id = shardSourceID("synth-fr-bare-street", { ...components, f: String(form), v: String(read) })
 
 			const canonical = {
 				raw,

package/src/shard-recipes/fr-lieudit.ts

@@ -29,13 +29,15 @@ import { join } from "node:path"
 
 import { extractBANAddrPoints } from "@mailwoman/ban/sdk"
 import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
+import { isPresent } from "@mailwoman/core/objects"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
 
-import { stableSourceID } from "../adapter.ts"
+import { stableSourceID } from "#adapters/utils"
+import type { CanonicalRow } from "#types"
+import { alignRow } from "#utils"
+
 import { decomposeFrStreet } from "../adapters/ban/street-decompose.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /**
@@ -156,7 +158,7 @@ function composeRaw(
 	}
 
 	lines.push(dependentLocality)
-	const cityLine = [postcode, locality].filter(Boolean).join(" ").trim()
+	const cityLine = [postcode, locality].filter(isPresent).join(" ").trim()
 
 	if (cityLine) {
 		lines.push(cityLine)

package/src/shard-recipes/fr-order.ts

@@ -30,15 +30,15 @@
  *   inlined the variant logic and never called it).
  */
 
-import { spawnSync } from "node:child_process"
-
+import { isPresent } from "@mailwoman/core/objects"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath } from "@mailwoman/core/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { stableSourceID } from "#adapters/utils"
+import type { CanonicalRow } from "#types"
+import { alignRow } from "#utils"
+
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /* oxlint-disable sister-software/no-unnamed-threshold -- the bare decimals below are weighted-sampler
    cutoffs, not thresholds: `const r = random()` followed by a cascade of `r < 0.4` branches IS the
@@ -72,28 +72,15 @@ interface FrTuple {
 }
 
 /**
- * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale; cap with `head` so the whole file
- * never has to be held in memory. Only keeps rows with a house_number (the shard's core signal) and a postcode
- * (required for reversed-order rendering to be meaningful).
+ * Stream FR tuples out of the cached OA zip. The countrywide extract is GB-scale, so this reads only as far as `limit`
+ * distinct tuples — the `break` closes the reader and releases the archive. Only keeps rows with a house_number (the
+ * shard's core signal) and a postcode (required for reversed-order rendering to be meaningful).
  */
-function readTuples(limit: number): FrTuple[] {
-	const maxLines = Math.max(limit * 8, 40_000) + 1
-
-	const r = spawnSync("bash", ["-c", `unzip -p "${SOURCE.zip}" "${SOURCE.csv}" | head -n ${maxLines}`], {
-		maxBuffer: 1024 * 1024 * 1024,
-		encoding: "buffer",
-	})
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${SOURCE.zip} (status ${r.status})`)
-
-		return []
-	}
-
+async function readTuples(limit: number): Promise<FrTuple[]> {
 	const tuples: FrTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(SOURCE.zip, SOURCE.csv)) {
 		if (tuples.length >= limit) break
 		const street = row.street ?? ""
 		const locality = row.city ?? ""
@@ -161,7 +148,7 @@ export const frOrderRecipe: ShardRecipe = {
 
 		// Over-read from the CSV so the dedup + filter pass can fill `count` rows.
 		const poolLimit = Math.max(count * 8, 40_000)
-		const pool = readTuples(poolLimit)
+		const pool = await readTuples(poolLimit)
 
 		console.error(`  ${SOURCE.csv}: ${pool.length} unique tuples (capped read)`)
 
@@ -211,7 +198,7 @@ export const frOrderRecipe: ShardRecipe = {
 			const { raw, components } = rendered
 
 			// Safety check: every component must appear verbatim in raw (alignment precondition).
-			const componentValues = Object.values(components).filter(Boolean) as string[]
+			const componentValues = Object.values(components).filter(isPresent)
 
 			if (!componentValues.every((v) => raw.includes(v))) {
 				skipped++

package/src/shard-recipes/german.ts

@@ -15,14 +15,13 @@
  *   docs/articles/evals/resolver-geo/2026-06-06-anchor-pilot.md (the order-artifact correction).
  */
 
-import { spawnSync } from "node:child_process"
-
 import { dataRootPath } from "@mailwoman/core/utils"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { synthesizeGermanRow, type LocaleBaseTuple } from "../synthesize-german.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { stableSourceID } from "#adapters/utils"
+import { synthesizeGermanRow, type LocaleBaseTuple } from "#synthesizers/german"
+import { alignRow } from "#utils"
+
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 /**
  * A German OA source (cached zip) + the Bundesland the file covers (OA's REGION column is empty for DE).
@@ -44,21 +43,13 @@ const SOURCES: GermanSource[] = [
 ]
 
 /**
- * Stream real German tuples out of a cached OA zip (buffered `unzip -p`).
+ * Stream real German tuples out of a cached OA zip.
  */
-function readGermanTuples(source: GermanSource): LocaleBaseTuple[] {
-	const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${source.zip} (status ${r.status})`)
-
-		return []
-	}
-
+async function readGermanTuples(source: GermanSource): Promise<LocaleBaseTuple[]> {
 	const tuples: LocaleBaseTuple[] = []
 	const seen = new Set<string>()
 
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(source.zip, source.csv)) {
 		const street = row.street ?? ""
 		const locality = row.city ?? ""
 
@@ -102,7 +93,7 @@ export const germanRecipe: ShardRecipe = {
 		const pool: LocaleBaseTuple[] = []
 
 		for (const s of SOURCES) {
-			const t = readGermanTuples(s)
+			const t = await readGermanTuples(s)
 
 			console.error(`  ${s.csv}: ${t.length} unique tuples`)
 

package/src/shard-recipes/house-venue.ts

@@ -10,7 +10,8 @@
  *   scripts/build-house-venue-shard.mjs.
  */
 
-import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "../synthesize-house-venue.ts"
+import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "#synthesizers/house-venue"
+
 import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"

package/src/shard-recipes/index.ts

@@ -9,6 +9,7 @@
  */
 
 import { anchorAbsorptionRecipe } from "./anchor-absorption.ts"
+import { bareCountryRecipe } from "./bare-country.ts"
 import { boundaryStressRecipe } from "./boundary-stress.ts"
 import { countryBalancedRecipe } from "./country-balanced.ts"
 import { czPcFirstPrepositionRecipe } from "./cz-pcfirst-preposition.ts"
@@ -27,12 +28,14 @@ import { noStreetLedRecipe } from "./no-street-led.ts"
 import { noStreetRecipe } from "./no-street.ts"
 import { poBoxCedexRecipe } from "./po-box-cedex.ts"
 import { poBoxRecipe } from "./po-box.ts"
+import { reviewedPostcodeTailRecipe } from "./reviewed-postcode-tail.ts"
 import type { ShardRecipe } from "./scaffold.ts"
 import { siBareVillageRecipe } from "./si-bare-village.ts"
-import { streetAffixRecipe } from "./street-affix.ts"
+import { streetAffixRecipe, suffixBoundaryRecipe } from "./street-affix.ts"
 import { streetBareRecipe } from "./street-bare.ts"
 import { streetRecipe } from "./street.ts"
 import { subVenueRecipe } from "./sub-venue.ts"
+import { trailingRegionRecipe } from "./trailing-region.ts"
 import { unitRecipe } from "./unit.ts"
 
 export * from "./scaffold.ts"
@@ -41,9 +44,13 @@ export * from "./scaffold.ts"
  * Every registered recipe, in display order.
  */
 const RECIPES: readonly ShardRecipe[] = [
+	bareCountryRecipe,
+	trailingRegionRecipe,
+	reviewedPostcodeTailRecipe,
 	streetRecipe,
 	streetBareRecipe,
 	streetAffixRecipe,
+	suffixBoundaryRecipe,
 	noStreetRecipe,
 	houseVenueRecipe,
 	poBoxRecipe,

package/src/shard-recipes/intersection.ts

@@ -34,18 +34,18 @@
  *   - `$MAILWOMAN_DATA_ROOT/oa-cache/us__il__cook.zip` (ZIP→city tails)
  */
 
-import { spawnSync } from "node:child_process"
-import { existsSync, readFileSync, writeFileSync } from "node:fs"
+import { existsSync, writeFileSync } from "node:fs"
 
 import type { DuckDBConnection } from "@duckdb/node-api"
 import type { ComponentTag } from "@mailwoman/core/types"
 import { dataRootPath, repoRootPath } from "@mailwoman/core/utils"
 import { JSONSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import type { CanonicalRow, LabeledRow } from "../types.ts"
-import { makeMulberry32, readCSVRecords, type ShardRecipe } from "./scaffold.ts"
+import { stableSourceID } from "#adapters/utils"
+import type { CanonicalRow, LabeledRow } from "#types"
+import { alignRow } from "#utils"
+
+import { makeMulberry32, readZippedCSVRecords, type ShardRecipe } from "./scaffold.ts"
 
 interface County {
 	fips: string
@@ -251,19 +251,11 @@ async function extractCrossings(
 /**
  * ZIP → majority city from the cached OA Cook-county CSV (real ZIP/city pairings).
  */
-function buildZipCityMap(): Map<string, string> {
-	const r = spawnSync("unzip", ["-p", OA_COOK.zip, OA_COOK.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" })
-
-	if (r.status !== 0) {
-		console.error(`  WARN: unzip failed for ${OA_COOK.zip} — city tails disabled`)
-
-		return new Map()
-	}
-
+async function buildZipCityMap(): Promise<Map<string, string>> {
 	const counts = new Map<string, Map<string, number>>()
 
 	// zip → Map(city → n)
-	for (const row of readCSVRecords(r.stdout)) {
+	for await (const row of readZippedCSVRecords(OA_COOK.zip, OA_COOK.csv)) {
 		const city = row.city ?? ""
 		const zip = row.postcode ?? ""
 
@@ -518,7 +510,7 @@ export const intersectionRecipe: ShardRecipe = {
 			throw new Error(`No crossings found — are the TIGER EDGES shapefiles present in ${edgesDir}?`)
 		}
 
-		const zipCity = opts.golden ? new Map<string, string>() : buildZipCityMap()
+		const zipCity = opts.golden ? new Map<string, string>() : await buildZipCityMap()
 
 		if (!opts.golden) {
 			console.error(`  zip→city map: ${zipCity.size} ZIPs (OA Cook)`)

package/src/shard-recipes/locale.ts

@@ -7,7 +7,7 @@
  *   OpenAddresses tuples for a `--country` (DE/FR/NL/IT/ES), renders each via
  *   {@link synthesizeLocaleRow} in BOTH orders (`--intl-fraction`, default 0.4 international / the
  *   rest country-native), aligns to BIO, and emits a labeled JSONL. Generate-mode: it STREAMS each
- *   source CSV (`unzip -p` for cached zips, plain `createReadStream` for extracted CSVs) and
+ *   source CSV (a streamed zip member for cached zips, plain `createReadStream` for extracted CSVs) and
  *   reservoir-samples to {@link RESERVOIR_CAP} (so FR/ES countrywide work in bounded memory), then
  *   draws `--count` rows from the pool with the passed `random`. Ported from
  *   scripts/build-locale-shard.mjs.
@@ -25,21 +25,23 @@
  *   country, so DE/FR emit streams are unchanged for a given seed.
  */
 
-import { spawn } from "node:child_process"
 import { createReadStream } from "node:fs"
 
 import { COUNTRY_SURFACE_FORMS } from "@mailwoman/codex/country"
+import { readZipEntry } from "@mailwoman/core/fs/zip"
+import { isPresent } from "@mailwoman/core/objects"
 import { dataRootPath } from "@mailwoman/core/utils"
 import { CSVSpliterator } from "spliterator"
 
-import { stableSourceID } from "../adapter.ts"
-import { alignRow } from "../align.ts"
-import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "../synthesize-german.ts"
+import { stableSourceID } from "#adapters/utils"
+import { type LocaleBaseTuple, type SynthesizedLocaleRow, synthesizeLocaleRow } from "#synthesizers/german"
+import { alignRow } from "#utils"
+
 import { makeMulberry32, type ShardRecipe } from "./scaffold.ts"
 
 /**
- * One per-country OA source part: either a cached `zip` + `csv` member (streamed via `unzip -p`) or an extracted plain
- * `path` (streamed via `createReadStream`). Both carry the standard OA header
+ * One per-country OA source part: either a cached `zip` + `csv` member (streamed out of the archive) or an extracted
+ * plain `path` (streamed via `createReadStream`). Both carry the standard OA header
  * (LON,LAT,NUMBER,STREET,UNIT,CITY,DISTRICT,REGION,POSTCODE,ID,HASH). An optional `region` fallback covers countries
  * whose REGION column is empty (DE — the Bundesland is implied by the per-state file).
  */
@@ -223,7 +225,7 @@ interface ColumnIndex {
 
 /**
  * Stream real tuples out of an OA source part and reservoir-sample to {@link RESERVOIR_CAP}. Reads the CSV row-by-row —
- * `unzip -p | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
+ * `readZipEntry | CSVSpliterator` for zip parts, `createReadStream | CSVSpliterator` for extracted parts (both bounded
  * memory) — and keeps a uniform random sample (Algorithm R) seeded by `rng`, separate from the emit loop's PRNG. NO
  * global dedup (a 25M-key Set would OOM; OA rows are near-unique). The city passes through {@link cleanCityNoise}; the
  * region falls back to `part.region` when the row's REGION cell is empty (DE).
@@ -231,21 +233,11 @@ interface ColumnIndex {
  * Exported for {@link locale.test.ts} — the CSV read path (quote handling, CRLF, region fallback) has no other test.
  */
 export async function readTuples(part: LocalePart, rng: () => number): Promise<LocaleBaseTuple[]> {
-	let input: NodeJS.ReadableStream
-
-	if (part.path) {
-		// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
-		// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
-		input = createReadStream(part.path)
-	} else {
-		const child = spawn("unzip", ["-p", part.zip!, part.csv!])
-
-		child.on("error", (err) => {
-			console.error(`  WARN: unzip failed for ${part.zip}: ${err.message}`)
-		})
-
-		input = child.stdout!
-	}
+	// No `encoding` on the file path — CSVSpliterator delimits raw bytes and decodes utf-8 itself; a string stream
+	// (from `{ encoding: "utf8" }`) would defeat its byte-range scanner.
+	const input: NodeJS.ReadableStream | AsyncIterable<Uint8Array> = part.path
+		? createReadStream(part.path)
+		: readZipEntry(part.zip!, part.csv!)
 
 	const get = (cells: string[], i: number): string => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "")
 	const reservoir: LocaleBaseTuple[] = []
@@ -302,7 +294,7 @@ export async function readTuples(part: LocalePart, rng: () => number): Promise<L
 			// pre-joined STREET column, so `cols.tipoVial === -1` and this is a no-op there.
 			const street =
 				cols.tipoVial >= 0
-					? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(Boolean).join(" ")
+					? [get(cells, cols.tipoVial), get(cells, cols.street)].filter(isPresent).join(" ")
 					: get(cells, cols.street)
 
 			const rawCity = get(cells, cols.city)

package/src/shard-recipes/no-street.ts

@@ -10,7 +10,8 @@
  *   scripts/build-no-street-shard.mjs.
  */
 
-import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "../synthesize-no-street.ts"
+import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "#synthesizers/no-street"
+
 import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
 
 const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"