@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/recipes/country-balanced.ts

@@ -3,59 +3,43 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `country-balanced` slice recipe — the BALANCED, MODEL-FIRST country-coverage slice (#464). The
- *   shipped model is STARVED on `country` (P=R=F1=0 on the homograph eval), so this fills the void
- *   the way the `unit` slice did, but built to AVOID over-firing "trailing token ⇒ country". Three
- *   ingredients, ported faithfully from scripts/build-country-slice-balanced.mjs:
- *
- *   1. Breadth/recall — real OA skeletons (US/DE/FR/IT/NL) with a country token in a varied surface form
- *        from `@mailwoman/codex/country` (canonical / endonym / ISO code), + ~30% country-ABSENT
- *        negatives (teach O-emission, the precision floor).
- *   2. Homograph CONTRAST pairs — each true country-name homograph (Georgia, Jordan, Lebanon, Mexico,
- *        Peru, Turkey) rendered BOTH ways: as `country` (foreign-city context) AND as the US
- *        `region`/`locality` (US-ZIP context). Teaches that the label is CONTEXTUAL, not
- *        positional.
- *   3. Code-as-region negatives — 2-letter codes that are both a US state abbrev and an ISO country code
- *        (CA/GA/IN/MA/PA/AL) in US-ZIP context → must read as `region`, never `country`.
- *
- *   `--golden` emits a held-out synthetic val over the VT (US) + Berlin (DE) holdouts. This is a
- *   `generate`-mode recipe that still reads REAL tuples off disk — `--count` bounds the OUTPUT, not
- *   the input. The passed `random` (the framework LCG) is consumed in the exact call order the
- *   legacy script used.
+ *   Builds country-label training rows from OpenAddresses tuples and codex country names. The mix includes rows with no
+ *   country and contrasts between country names and US place names. `--golden` reads held-out sources.
  */
 
 import type { ComponentTag } from "@mailwoman/codex/component"
 import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country"
 import { dataRootPath } from "@mailwoman/core/data-root"
+import { stringifyJSON } from "@mailwoman/core/json"
 import { isPresent } from "@mailwoman/core/objects"
+import { sample } from "@mailwoman/core/random"
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 import type { PathBuilderLike } from "path-ts"
 
-import { stableSourceID } from "#adapters/utils"
-import { readOATuples, type CorpusRecipe } from "#recipes/scaffold"
-import { pick } from "#synthesizers/utils"
+import { stableSourceID } from "#adapters/source-id"
+import { readOATuples, requireRegister, type CorpusRecipe } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 import type { CanonicalRow } from "#types"
 import { alignRow } from "#utils"
 
-// v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
-// ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
-// so endonyms/abbrevs ("Deutschland","USA","NL") get strong signal.
-
+// Country names are drawn without regard to the address's own country.
 const COUNTRY_FORM_POOL = (() => {
-	const surface = Object.values(COUNTRY_SURFACE_FORMS).flat() // endonyms + abbrevs + canonical (curated)
+	// `surface` holds curated endonyms and abbreviations.
+	// `names` holds canonical English names.
+	const surface = Object.values(COUNTRY_SURFACE_FORMS).flat()
 	const names = [...CountryNames]
 
-	// all ~249 ISO canonical English names (breadth)
 	return { surface, names }
 })()
 
 /**
- * Negatives: rows with NO country token → teach golden precision.
+ * The share of address rows rendered with no country.
  */
 const COUNTRY_ABSENT_PROB = 0.3
 
 /**
- * A cached OpenAddresses extract + the implied iso2/region/render-order.
+ * One OpenAddresses archive and how to render its rows.
  */
 interface CountrySource {
 	zip: PathBuilderLike
@@ -66,8 +50,7 @@ interface CountrySource {
 }
 
 /**
- * Multi-locale OA sources. region = implied admin where the extract is single-region (US states, DE Saxony);
- * countrywide extracts (FR/IT/NL) read region from the CSV when present.
+ * The training sources.
  */
 const SOURCES: readonly CountrySource[] = [
 	{
@@ -106,8 +89,6 @@ const SOURCES: readonly CountrySource[] = [
 		region: "",
 		order: "fr",
 	},
-	// ES uses the Spanish IGN schema, not the OA standard columns — skipped here (codex still recognizes
-	// "España"/"Spain"). A dedicated IGN adapter is a follow-up.
 	{
 		zip: dataRootPath("oa-cache", "it__countrywide.zip"),
 		csv: "it/countrywide.csv",
@@ -125,7 +106,7 @@ const SOURCES: readonly CountrySource[] = [
 ]
 
 /**
- * Held-out for --golden: Vermont (US holdout) + Berlin (DE holdout) — geographic split, never trained.
+ * The held-out sources that `--golden` reads.
  */
 const EVAL_SOURCES: readonly CountrySource[] = [
 	{
@@ -139,7 +120,7 @@ const EVAL_SOURCES: readonly CountrySource[] = [
 ]
 
 /**
- * A real tuple read out of a cached OA zip (+ the source's iso2/render-order).
+ * An address tuple with its source country and rendering order.
  */
 interface CountryTuple {
 	house_number: string
@@ -152,8 +133,7 @@ interface CountryTuple {
 }
 
 /**
- * The countrywide extracts (FR/IT/NL) are GB-scale, so this reads only as far as `limit` distinct tuples: the `break`
- * closes the reader, which releases the archive without inflating the rest of the member.
+ * Reads up to `limit` tuples from one source archive.
  */
 async function readTuples(source: CountrySource, limit: number): Promise<CountryTuple[]> {
 	return readOATuples(source, {
@@ -167,25 +147,27 @@ async function readTuples(source: CountrySource, limit: number): Promise<Country
 	})
 }
 
-// After the absent draw: 60% curated surface forms (endonym/abbrev variety), 40% broad ISO canonical names.
+// The share of country draws taken from the curated surface forms instead of canonical names.
 const SURFACE_FORM_SHARE = 0.6
 
 /**
- * Pick a country token from the BROAD pool, or null (a country-absent negative). v2.
+ * Picks a country name, or returns `null` for a row with no country.
  */
 function pickCountry(random: () => number): string | null {
-	if (random() < COUNTRY_ABSENT_PROB) return null // negative — teaches "trailing token != always country"
+	if (random() < COUNTRY_ABSENT_PROB) return null
 	const pool = random() < SURFACE_FORM_SHARE ? COUNTRY_FORM_POOL.surface : COUNTRY_FORM_POOL.names
 
-	return pick(pool, random)
+	return sample(pool, random)
 }
 
-// Country-bearing rows: 80% full, 12% full-nl, 8% bare.
+// A draw below `FULL_CUTOFF` selects the comma form.
+// A draw below `FULL_NEWLINE_CUTOFF` selects the newline form.
+// A higher draw selects the bare form without region or postcode.
 const FULL_CUTOFF = 0.8
 const FULL_NEWLINE_CUTOFF = 0.92
 
 /**
- * Render the address body in native-ish order. `country` null → a country-ABSENT negative row.
+ * Renders an address in its source order, followed by the country when one is given.
  */
 function renderCountry(
 	random: () => number,
@@ -209,17 +191,15 @@ function renderCountry(
 		const regPc = [reg, pc].filter(isPresent).join(" ")
 		body = `${hn} ${street}, ${loc}${regPc ? ", " + regPc : ""}`
 	} else if (order === "fr") {
-		// French is NUMBER-street, postcode-city: "84 Route de la Fontaine, 75008 Paris".
+		// France uses number, street, postcode, locality.
 		body = `${hn} ${street}, ${[pc, loc].filter(isPresent).join(" ")}`
 	} else {
-		// DE/IT/NL: street-number, postcode-city: "Pariser Platz 1, 10117 Berlin".
+		// Germany, Italy and the Netherlands use street, number, postcode, locality.
 		const pcCity = [pc, loc].filter(isPresent).join(" ")
 		body = `${street} ${hn}, ${pcCity}`
 	}
 
 	if (!country) {
-		// Negative: a normal address, NO country token/component. Teaches that a trailing region/city/
-		// postcode is NOT a country (counters the v1 golden over-firing).
 		return { fmt: "negative", raw: body, components }
 	}
 
@@ -238,10 +218,7 @@ function renderCountry(
 	}
 }
 
-// ── Homograph contrast (the model-first addition) ───────────────────────────────────────────────
-// True country-name homographs: the surface form is BOTH a country AND a US state/locality. Rendering
-// each BOTH ways (foreign-city → country; US-ZIP → region/locality) is what teaches the CONTEXTUAL
-// distinction. role: how the surface reads in US context.
+// A name shared by a country and a US state or locality.
 interface Homograph {
 	surface: string
 	iso2: string
@@ -288,20 +265,21 @@ const HOMOGRAPHS: readonly Homograph[] = [
 	},
 ]
 
-// 2-letter codes that are BOTH a US state abbrev AND an ISO country code → must read as region in US ctx.
+// A US state code that is also an ISO country code.
 interface AbbrevRegion {
 	code: string
 	localities: readonly string[]
 	postcodes: readonly string[]
 }
 
+// The paired countries are Canada, Georgia, India, Morocco, Panama and Albania.
 const ABBREV_REGIONS: readonly AbbrevRegion[] = [
-	{ code: "CA", localities: ["Los Angeles", "Sacramento", "San Diego"], postcodes: ["90012", "95814", "92101"] }, // California / Canada
-	{ code: "GA", localities: ["Atlanta", "Savannah", "Macon"], postcodes: ["30309", "31401", "31201"] }, // Georgia(US) / Georgia
-	{ code: "IN", localities: ["Indianapolis", "Fort Wayne"], postcodes: ["46204", "46802"] }, // Indiana / India
-	{ code: "MA", localities: ["Boston", "Worcester"], postcodes: ["02108", "01608"] }, // Massachusetts / Morocco
-	{ code: "PA", localities: ["Philadelphia", "Pittsburgh"], postcodes: ["19103", "15222"] }, // Pennsylvania / Panama
-	{ code: "AL", localities: ["Birmingham", "Montgomery"], postcodes: ["35203", "36104"] }, // Alabama / Albania
+	{ code: "CA", localities: ["Los Angeles", "Sacramento", "San Diego"], postcodes: ["90012", "95814", "92101"] },
+	{ code: "GA", localities: ["Atlanta", "Savannah", "Macon"], postcodes: ["30309", "31401", "31201"] },
+	{ code: "IN", localities: ["Indianapolis", "Fort Wayne"], postcodes: ["46204", "46802"] },
+	{ code: "MA", localities: ["Boston", "Worcester"], postcodes: ["02108", "01608"] },
+	{ code: "PA", localities: ["Philadelphia", "Pittsburgh"], postcodes: ["19103", "15222"] },
+	{ code: "AL", localities: ["Birmingham", "Montgomery"], postcodes: ["35203", "36104"] },
 ]
 
 const STREET_POOL: readonly string[] = [
@@ -319,12 +297,11 @@ const STREET_POOL: readonly string[] = [
 
 const houseNo = (random: () => number): string => String(1 + Math.floor(random() * 998))
 
-// Country-surface homograph rows: 60% carry a street line, 40% are a bare `city, surface`.
+// The share of country-reading homograph rows that include a street address.
 const HOMOGRAPH_WITH_STREET_SHARE = 0.6
 
 /**
- * A homograph CONTRAST row: ~half render the surface as `country` (foreign city), half as the US `region`/`locality`
- * (US ZIP, NO country). Returns iso2 for provenance.
+ * Renders a homograph as either the country or the US place, with the matching country code.
  */
 function renderHomograph(random: () => number): {
 	fmt: string
@@ -332,13 +309,13 @@ function renderHomograph(random: () => number): {
 	components: Partial<Record<ComponentTag, string>>
 	iso2: string
 } {
-	const h = pick(HOMOGRAPHS, random)
+	const h = sample(HOMOGRAPHS, random)
 
 	const hn = houseNo(random),
-		street = pick(STREET_POOL, random)
+		street = sample(STREET_POOL, random)
 
 	if (random() < 0.5) {
-		const city = pick(h.cities, random)
+		const city = sample(h.cities, random)
 		const withStreet = random() < HOMOGRAPH_WITH_STREET_SHARE
 		const raw = withStreet ? `${hn} ${street}, ${city}, ${h.surface}` : `${city}, ${h.surface}`
 
@@ -349,10 +326,9 @@ function renderHomograph(random: () => number): {
 		return { fmt: "homograph-country", raw, components, iso2: h.iso2 }
 	}
 
-	const pc = pick(h.us.postcodes, random)
+	const pc = sample(h.us.postcodes, random)
 
 	if (h.us.role === "region") {
-		// surface is the US STATE: "123 Oak Ave, Atlanta, Georgia 30309" → region, no country
 		return {
 			fmt: "homograph-us-region",
 			raw: `${hn} ${street}, ${h.us.locality}, ${h.surface} ${pc}`,
@@ -361,7 +337,6 @@ function renderHomograph(random: () => number): {
 		}
 	}
 
-	// surface is the US CITY: "123 Oak Ave, Lebanon, TN 37087" → locality, no country
 	return {
 		fmt: "homograph-us-locality",
 		raw: `${hn} ${street}, ${h.surface}, ${h.us.region} ${pc}`,
@@ -371,7 +346,7 @@ function renderHomograph(random: () => number): {
 }
 
 /**
- * An abbrev-as-region negative: "123 Main St, Los Angeles, CA 90012" → region CA, NO country.
+ * Renders a US address whose state code is also a country code.
  */
 function renderAbbrevRegion(random: () => number): {
 	fmt: string
@@ -379,12 +354,12 @@ function renderAbbrevRegion(random: () => number): {
 	components: Partial<Record<ComponentTag, string>>
 	iso2: string
 } {
-	const a = pick(ABBREV_REGIONS, random)
+	const a = sample(ABBREV_REGIONS, random)
 
 	const hn = houseNo(random),
-		street = pick(STREET_POOL, random),
-		locality = pick(a.localities, random),
-		postcode = pick(a.postcodes, random)
+		street = sample(STREET_POOL, random),
+		locality = sample(a.localities, random),
+		postcode = sample(a.postcodes, random)
 
 	return {
 		fmt: "abbrev-region",
@@ -395,31 +370,30 @@ function renderAbbrevRegion(random: () => number): {
 }
 
 /**
- * Share of rows that are homograph contrast pairs.
+ * The share of rows that are homograph contrasts.
  */
 const HOMOGRAPH_FRAC = 0.22
 /**
- * Share that are code-as-region negatives (cumulative with HOMOGRAPH_FRAC)
+ * The share of rows that are state-code contrasts.
  */
 const ABBREV_FRAC = 0.08
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * The country-balanced recipe registered with the corpus builder.
  */
 export const countryBalancedRecipe: CorpusRecipe = {
 	name: "country-balanced",
 	description: "Balanced model-first country rows (#464): OA skeletons + ISO surface forms + homograph contrast pairs",
 	mode: "generate",
-	options: [{ flag: "--golden", description: "Emit the held-out VT+Berlin eval slice" }],
+	options: [{ flag: "--golden", description: "Emit the held-out VT+Berlin eval set" }],
 	async run(opts, write) {
 		if (opts.count == null) throw new Error("country-balanced recipe requires --count <N>")
 		const count = opts.count
-		// Legacy build-country-slice-balanced.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
 		const random = makeMulberry32(opts.seed)
-		const source = opts.sourceName ?? "synth-country"
+		const source = opts.sourceName ?? defaultRecipeSource("synth-country")
 		const sources = opts.golden ? EVAL_SOURCES : SOURCES
-		const perSource = Math.ceil((count * 3) / sources.length) // over-read; balance locales
+		// A threefold read of the target leaves room for skipped rows.
+		const perSource = Math.ceil((count * 3) / sources.length)
 
 		const pool: CountryTuple[] = []
 
@@ -443,8 +417,6 @@ export const countryBalancedRecipe: CorpusRecipe = {
 		const N = pool.length
 
 		while (emitted < count && guard++ < count * 8) {
-			// Mix three row types: homograph contrast (the distinction), code-as-region negatives, and the
-			// breadth/recall main path (random ISO form on an OA skeleton, ~30% country-absent).
 			const roll = random()
 			let rendered: { fmt: string; raw: string; components: Partial<Record<ComponentTag, string>> }
 			let rowISO2: string
@@ -459,7 +431,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
 				rowISO2 = a.iso2
 			} else {
 				const t = pool[Math.floor(random() * N)]!
-				const country = pickCountry(random) // may be null → a country-absent negative row
+				const country = pickCountry(random)
 				rendered = renderCountry(random, t, country)
 				rowISO2 = t.iso2
 
@@ -474,7 +446,7 @@ export const countryBalancedRecipe: CorpusRecipe = {
 			const localeTag = rowISO2 === "US" ? "en-US" : `${rowISO2.toLowerCase()}-${rowISO2}`
 
 			if (opts.golden) {
-				write(JSON.stringify({ raw, components, country: rowISO2 }) + "\n")
+				write(stringifyJSON({ raw, components, country: rowISO2 }))
 
 				emitted++
 
@@ -500,7 +472,15 @@ export const countryBalancedRecipe: CorpusRecipe = {
 				continue
 			}
 
-			write(JSON.stringify({ ...aligned.row, synth_method: "country", synth_base_id: null }) + "\n")
+			write(
+				stringifyJSON({
+					...aligned.row,
+					recipe: "country",
+					base_source_id: null,
+					register: requireRegister(opts, "country-balanced"),
+					surface: SurfaceOrigin.Composed,
+				})
+			)
 
 			emitted++
 		}

package/lib/recipes/house-venue.ts

@@ -3,11 +3,11 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `house-venue` slice recipe — synthetic house_number+venue+street co-occurrence rows: tuples →
+ *   `house-venue` recipe — synthetic house_number+venue+street co-occurrence rows: tuples →
  *   {@link synthesizeHouseVenueRow} → aligned LabeledRow. The v0.6.3 corrective companion to the
- *   no-street slice: every row carries BOTH house_number AND venue, restoring the house_number
- *   signal that no-street's distributional shift cost the model. Ported from
- *   scripts/build-house-venue-slice.mjs.
+ *   no-street recipe: every row includes both house_number and venue, restoring the house_number
+ *   signal that no-street's distributional shift cost the model. Ported from the root build script
+ *   it replaced.
  */
 
 import { makeLcg } from "@mailwoman/core/utils"
@@ -15,15 +15,32 @@ import { makeLcg } from "@mailwoman/core/utils"
 import {
 	alignAndWrite,
 	readTuples,
-	sliceSourceID,
+	recipeSourceID,
 	SYNTHETIC_TUPLE_LICENSE as LICENSE,
 	type CorpusRecipe,
 } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
 import { synthesizeHouseVenueRow, type HouseVenueBaseTuple } from "#synthesizers/house-venue"
+import { SurfaceOrigin } from "#types"
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * The venue name and house number are drawn and joined to a tuple's admin components.
+ *
+ * No register asserts that this venue stands at this number, so the row teaches the
+ * co-occurrence shape rather than reporting a published record.
+ */
+const HOUSE_VENUE_PROVENANCE = {
+	register: null,
+	surface: SurfaceOrigin.Invented,
+	// An invented row has no underlying record to name.
+	baseSourceID: null,
+}
+
+/**
+ * Recipe registered with the corpus builder.
+ *
+ * See the file header for the parse behavior it exercises.
+ * See `description` below for the generated surface form.
  */
 export const houseVenueRecipe: CorpusRecipe = {
 	name: "house-venue",
@@ -31,9 +48,9 @@ export const houseVenueRecipe: CorpusRecipe = {
 	mode: "tuples",
 	async run(opts, write) {
 		if (!opts.input) throw new Error("house-venue recipe requires --input <tuples.jsonl>")
-		// Legacy build-house-venue-slice.mjs seeded the LCG via makeRandom(opts.seed) (s = seed).
+		// The root build script this recipe replaced seeded the LCG via makeRandom(opts.seed) (s = seed).
 		const random = makeLcg(opts.seed)
-		const source = opts.sourceName ?? "synth-house-venue"
+		const source = opts.sourceName ?? defaultRecipeSource("synth-house-venue")
 		let read = 0
 		let emitted = 0
 		let skipped = 0
@@ -41,8 +58,8 @@ export const houseVenueRecipe: CorpusRecipe = {
 		for await (const tuple of readTuples(opts.input)) {
 			read++
 
-			// FR renders without a region (postcode-before-locality tail — the run-2 contingency), so
-			// an empty region is valid there and stays required everywhere else.
+			// FR renders without a region (postcode-before-locality tail — the run-2 contingency),
+			// so an empty region is valid there and stays required everywhere else.
 			if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && tuple.country !== "FR")) {
 				skipped++
 
@@ -66,7 +83,7 @@ export const houseVenueRecipe: CorpusRecipe = {
 						country: tuple.country,
 						locale: synth.locale,
 						source,
-						source_id: sliceSourceID(source, {
+						source_id: recipeSourceID(source, {
 							locality: tuple.locality,
 							region: tuple.region,
 							postcode: tuple.postcode,
@@ -77,7 +94,8 @@ export const houseVenueRecipe: CorpusRecipe = {
 						corpus_version: "0.4.0",
 						license: LICENSE,
 					},
-					synth.template
+					synth.template,
+					HOUSE_VENUE_PROVENANCE
 				)
 
 				if (ok) {