@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/recipes/nl-postcode.ts

@@ -1,117 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `nl-postcode` — the Dutch full-form postcode slice (#924). The model reads the DIGITS-FIRST NL
- *   postcode "1012 LG" as a house number + a 2-letter street ("1012 LG Amsterdam" → house_number
- *   1012 / street "LG" / locality Amsterdam), and that spurious street context then pulls the
- *   locality into the US situs tier (Amsterdam → Amsterdam, NY). Letters-first postcodes (UK
- *   "SW1A 1AA") parse natively; the `\d{4} [A-Z]{2}` shape does not, and the soft query-shape prior
- *   (0.9 log-odds) can't overcome the strong house-number reading of a leading 4-digit token.
- *
- *   This is the model-first fix as DATA (the #723/#901 discipline — teach the boundary, don't
- *   override the decoder): real NL (street, number, postcode, city) tuples in the orders Dutch
- *   addresses actually use, with the full postcode tagged as ONE postcode span. Both the SPACED
- *   ("1012 LG", the failing form — a 2-token span) and UNSPACED ("1012LG", 1 token) forms are
- *   emitted so the model learns the digits-first postcode regardless of spacing; the three orders
- *   keep polarity balanced (the v1.9.9 lesson).
- */
-
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
-
-/**
- * "1012LG" → "1012 LG". The tuples carry the unspaced OA form; the spaced form is the failing case.
- */
-function spacePostcode(pc: string): string {
-	return pc.replace(/^(\d{4})([A-Z]{2})$/, "$1 $2")
-}
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const nlPostcodeRecipe: CorpusRecipe = {
-	name: "nl-postcode",
-	description:
-		"NL full-form postcode (#924): teach '\\d{4} [A-Z]{2}' = postcode, not house#+street — spaced + unspaced, 3 orders",
-	mode: "tuples",
-	async run(opts, write) {
-		makeMulberry32(opts.seed)
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-
-		for await (const t of readTuples(opts.input!)) {
-			read++
-			const street = String(t.street ?? "").trim()
-			const city = String(t.locality ?? "").trim()
-			const number = String(t.number ?? "").trim()
-
-			const rawPostcode = String(t.postcode ?? "")
-				.trim()
-				.toUpperCase()
-				.replaceAll(/\s+/g, "")
-
-			if (!street || !city || !number || !/^\d{4}[A-Z]{2}$/.test(rawPostcode)) {
-				skipped++
-
-				continue
-			}
-
-			// Spacing rotates so the model sees BOTH the failing spaced form and the unspaced form; the
-			// components.postcode value MUST match the raw form so alignment tags the right span.
-			const spaced = read % 2 === 0
-			const postcode = spaced ? spacePostcode(rawPostcode) : rawPostcode
-
-			// The three orders Dutch addresses use. `street number, postcode city` is canonical; the
-			// pc-first form is where the leading digits most strongly mis-read as a house number.
-			const order = read % 3
-			let raw: string
-
-			if (order === 0) {
-				raw = `${street} ${number}, ${postcode} ${city}`
-			} else if (order === 1) {
-				raw = `${postcode} ${city}, ${street} ${number}`
-			} else {
-				raw = `${city}, ${postcode}, ${street} ${number}`
-			}
-
-			const components: Record<string, string> = {
-				street,
-				house_number: number,
-				postcode,
-				locality: city,
-			}
-
-			const source_id = sliceSourceID("synth-nl-postcode", {
-				...components,
-				o: String(order),
-				s: spaced ? "1" : "0",
-				v: String(read),
-			})
-
-			const canonical = {
-				raw,
-				components,
-				country: "NL",
-				locale: "nl-NL",
-				source: "synth-nl-postcode",
-				source_id,
-				corpus_version: "0.10.0",
-				license:
-					"Synthetic — nl-postcode; (street, number, postcode, city) from OpenAddresses NL (per-source attribution in the model card)",
-			}
-
-			if (alignAndWrite(write, canonical, "nl-postcode")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		return { read, emitted, skipped }
-	},
-}

package/lib/recipes/no-fragment.ts

@@ -1,213 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `no-fragment` — the Norwegian house-number-licence change (Track B, 2026-07-16). The mirror of
- *   `fr-fragment`, which earned +50pp on the same defect shape in French.
- *
- *   WHY THIS EXISTS AND `no-street-led` DOES NOT SUFFICE. Board 3 (the NO digit board) measured that
- *   `synth-no-street-led`'s three forms — all carrying postcode+city — are ALREADY at 0.940-0.968 on
- *   a model with zero Norwegian rows. The headroom is in the forms that slice never emits:
- *
- *     bare-street-hn   "Hallingrudveien 32"     0.693   — no postcode competing, still fails 31%
- *     slash-hn         "Øvrabø 124/1"           0.650   — cadastral gnr/bnr, ONE component
- *
- *   THE MECHANISM the board exposed: `Hallingrudveien 32` -> locality + postcode, while
- *   `Hallingrudveien 32, 3370 Vikersund` parses perfectly. The street loses its street reading and
- *   the digit loses its anchor TOGETHER. That is Track A's bare-street LICENCE in Norwegian — the
- *   model will not read a street without its postcode/locality partner — not a digit-ownership prior.
- *   fr-fragment fixed exactly this in French by teaching the street WITHOUT its partners.
- *
- *   THE COUNTER-DISTRIBUTION IS THE POINT (fr-fragment's lesson, and board 2's bare-locality guard).
- *   Teaching bare `{street} {number}` alone lets the model satisfy every row by flipping its default
- *   from "bare toponym -> locality" to "bare toponym -> street", trading one broken prior for
- *   another. Two counter-classes hold the line:
- *     - bare LOCALITIES (no street) so "bare -> street" is not free.
- *     - bare POSTCODES so the model does not learn to stop emitting postcode to win the digit — the
- *       board 3 bare-pc negative class (1.000) must HOLD.
- *
- *   SLASH HAZARD, pinned deliberately: NO `124/1` is ONE house_number (cadastral gnr/bnr). AU
- *   `12/345` is TWO (unit + house_number). This slice teaches the Norwegian reading; a future AU
- *   intra-word-split slice (B5) must not generalize over it. The two are locale-restricted by design.
- *
- *   SPLIT: `--exclude-surfaces` REQUIRED (throws otherwise) — the digit board's reserved surface
- *   list. Diacritic-KEEPING normalizer, matching the board (see the norm docstring).
- */
-
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-import { TextSpliterator } from "spliterator"
-
-import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
-
-/**
- * Title-case a Kartverket ALL-CAPS locality (HELLVIK -> Hellvik); #690, all-caps is OOD.
- */
-const titleNO = (value: string): string =>
-	value
-		.split(/\s+/)
-		.map((w) => (w ? w[0]!.toUpperCase() + w.slice(1).toLowerCase() : w))
-		.join(" ")
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const noFragmentRecipe: CorpusRecipe = {
-	name: "no-fragment",
-	description:
-		"NO street fragments — the house-number-licence change (Track B): '«st» «n»' / bare «st» with NO postcode partner, + bare-locality & bare-postcode counters",
-	mode: "tuples",
-	options: [
-		{
-			flag: "--exclude-surfaces <path>",
-			description:
-				"REQUIRED. The NO digit board's reserved surface list (mailwoman/eval-harness/fixtures/no-digits.surfaces.txt).",
-		},
-		{
-			flag: "--bare-street-prob <n>",
-			description: "Share of street rows emitted as a BARE street, no number (default 0.30 — the pure licence signal)",
-		},
-		{
-			flag: "--counter-prob <n>",
-			description: "Share of ALL rows that are counter-distribution (bare locality OR bare postcode) (default 0.30)",
-		},
-		{
-			flag: "--long-number-boost <n>",
-			description:
-				"knob 3: emit N copies of each street+number row whose number has >= --long-number-min-digits digits — oversample the failing long-number class the model calls a postcode (default 1 = no boost)",
-		},
-		{
-			flag: "--long-number-min-digits <n>",
-			description: "knob 3: minimum digit count for a number to be 'long' and boosted (default 3)",
-		},
-	],
-	async run(opts, write) {
-		const random = makeMulberry32(opts.seed)
-		const excludePath = opts.excludeSurfaces
-
-		if (!excludePath) {
-			throw new Error(
-				"no-fragment: --exclude-surfaces is REQUIRED. Pass the NO digit board's reserved list " +
-					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this slice trains on its own eval set. " +
-					"Source-disjoint by street SURFACE is the split discipline; there is no safe default."
-			)
-		}
-
-		const excluded = new Set<string>()
-
-		for await (const line of TextSpliterator.fromAsync(excludePath)) {
-			const trimmed = line.trim()
-
-			if (trimmed && !trimmed.startsWith("#")) {
-				excluded.add(trimmed)
-			}
-		}
-
-		if (!excluded.size) throw new Error(`no-fragment: --exclude-surfaces "${excludePath}" listed no surfaces`)
-
-		const bareStreetProb = opts.bareProb ?? 0.3
-		const counterProb = opts.counterProb ?? 0.3
-		const longNumberBoost = Math.max(1, Math.floor(opts.longNumberBoost ?? 1))
-		const longNumberMinDigits = opts.longNumberMinDigits ?? 3
-
-		// Harvested from the tuples — every NO row carries its locality and postcode, so the two
-		// counter-classes need no second source.
-		const localities = new Set<string>()
-		const postcodes = new Set<string>()
-
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-		let contaminated = 0
-		let emitSeq = 0
-
-		const emit = (raw: string, components: Record<string, string>, klass: string): void => {
-			// emitSeq keeps every emit distinct — knob 3 emits N copies of one long-number row, and
-			// (components, read) alone would collide their source_id and let downstream dedup drop the boost.
-			const source_id = sliceSourceID("synth-no-fragment", { ...components, k: klass, v: `${read}:${emitSeq++}` })
-
-			const canonical = {
-				raw,
-				components,
-				country: "NO",
-				locale: "nb-NO",
-				source: "synth-no-fragment",
-				source_id,
-				corpus_version: "0.11.0",
-				license: "Synthetic — no-fragment; (street, number, postcode, city) from OpenAddresses NO / Kartverket",
-			}
-
-			if (alignAndWrite(write, canonical, "no-fragment")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		for await (const tuple of readTuples(opts.input!)) {
-			read++
-			const street = String(tuple.street ?? "").trim()
-			const locality = titleNO(String(tuple.locality ?? "").trim())
-			const number = String(tuple.number ?? "").trim()
-			const postcode = String(tuple.postcode ?? "").trim()
-
-			if (locality) {
-				localities.add(locality)
-			}
-
-			if (postcode) {
-				postcodes.add(postcode)
-			}
-
-			if (!street) {
-				skipped++
-
-				continue
-			}
-
-			// THE SPLIT. A surface on the digit board never enters training.
-			if (excluded.has(foldNOSurface(street))) {
-				contaminated++
-
-				continue
-			}
-
-			// COUNTER-DISTRIBUTION — drawn from the harvested pools, not this row's street. Half bare
-			// localities (so "bare -> street" is not free), half bare postcodes (so the model does not
-			// stop emitting postcode to win the digit — board 3's bare-pc must hold).
-			if (random() < counterProb) {
-				if (random() < 0.5 && localities.size) {
-					const loc = [...localities][Math.floor(random() * localities.size)]!
-
-					emit(loc, { locality: loc }, "counter-bare-locality")
-				} else if (postcodes.size) {
-					const pc = [...postcodes][Math.floor(random() * postcodes.size)]!
-
-					emit(pc, { postcode: pc }, "counter-bare-postcode")
-				}
-
-				continue
-			}
-
-			// THE SIGNAL. A street with NO postcode/locality partner. Either bare, or street+number —
-			// both are the forms board 3 measured as the headroom (bare-street-hn 0.693, slash-hn 0.650).
-			if (!number || random() < bareStreetProb) {
-				emit(street, { street }, "bare-street")
-			} else {
-				const klass = number.includes("/") ? "slash-hn" : "street-hn"
-				// knob 3: the failing class is street + LONG number (Leppdalsvegen 1285 -> postcode). The
-				// digit count, not the slash, is what tips the length prior toward postcode. Oversample
-				// those rows to fight the prior with volume and teach the street/number boundary directly.
-				const digits = (number.match(/\d/g) ?? []).length
-				const copies = digits >= longNumberMinDigits ? longNumberBoost : 1
-
-				for (let c = 0; c < copies; c++) {
-					emit(`${street} ${number}`, { street, house_number: number }, klass)
-				}
-			}
-		}
-
-		return { read, emitted, skipped, contaminated }
-	},
-}

package/lib/recipes/no-street-led.ts

@@ -1,135 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `no-street-led` — the Norwegian street-led slice, third orthography of the #901
- *   leading-name-boundary family. The post-#920 NO row-read measured the residual as
- *   ORDER-SENSITIVE decode: street-led forms ("Tangavegen 40, 5620 Tørvikbygd") carry a 30%
- *   residual vs pc-first's 7% — the diacritic street head (…vegen/…veien with ø/å/æ) is the same
- *   leading-name-before-comma boundary the FR bare-street and SI village forms exercise, in a
- *   third orthography. All three real orders cycle per tuple (balanced polarity — the v1.9.9
- *   lesson: a one-order gradient loses at convergence):
- *
- *   1. canonical  "«st» «n», «pc» «city»"   (the 30% residual class — the lesson)
- *   2. city-first "«city», «pc», «st» «n»"
- *   3. pc-first   "«pc» «city», «st» «n»"   (the 7% floor — the anchor the others converge to)
- */
-
-import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
-import { TextSpliterator } from "spliterator"
-
-import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const noStreetLedRecipe: CorpusRecipe = {
-	name: "no-street-led",
-	description: "NO street-led boundary form (#901 family): '«st» «n», «pc» «city»' — diacritic street heads",
-	mode: "tuples",
-	options: [
-		{
-			flag: "--exclude-surfaces <path>",
-			description:
-				"REQUIRED. The NO digit board's reserved surface list " +
-				"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt); every listed street is skipped.",
-		},
-	],
-	async run(opts, write) {
-		makeMulberry32(opts.seed)
-
-		// THE SPLIT (ported from fr-fragment, #727 T2). Without it this slice trains on all 10,697
-		// NO surfaces, 1,952 of which the digit board reserves — so a Norway retrain would grade
-		// memorization of `Hallingrudveien` while claiming to measure the boundary form. There is no
-		// safe default: source-disjoint by street SURFACE is the discipline, so the flag throws.
-		const excludePath = opts.excludeSurfaces
-
-		if (!excludePath) {
-			throw new Error(
-				"no-street-led: --exclude-surfaces is REQUIRED. Pass the NO digit board's reserved list " +
-					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this slice trains on its own eval set. " +
-					"Source-disjoint by street SURFACE is the split discipline; there is no safe default."
-			)
-		}
-
-		const excluded = new Set<string>()
-
-		for await (const line of TextSpliterator.fromAsync(excludePath)) {
-			const trimmed = line.trim()
-
-			if (trimmed && !trimmed.startsWith("#")) {
-				excluded.add(trimmed)
-			}
-		}
-
-		if (!excluded.size) throw new Error(`no-street-led: --exclude-surfaces "${excludePath}" listed no surfaces`)
-
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-		let contaminated = 0
-
-		for await (const t of readTuples(opts.input!)) {
-			read++
-			const street = String(t.street ?? "").trim()
-			const city = String(t.locality ?? "").trim()
-			const number = String(t.number ?? "").trim()
-			const postcode = String(t.postcode ?? "").trim()
-
-			if (!street || !city || !number || !postcode) {
-				skipped++
-
-				continue
-			}
-
-			// A surface on the digit board never enters training.
-			if (excluded.has(foldNOSurface(street))) {
-				contaminated++
-
-				continue
-			}
-
-			const order = read % 3
-			let raw: string
-
-			const components: Record<string, string> = {
-				street,
-				house_number: number,
-				postcode,
-				locality: city,
-			}
-
-			if (order === 0) {
-				raw = `${street} ${number}, ${postcode} ${city}`
-			} else if (order === 1) {
-				raw = `${city}, ${postcode}, ${street} ${number}`
-			} else {
-				raw = `${postcode} ${city}, ${street} ${number}`
-			}
-
-			const source_id = sliceSourceID("synth-no-street-led", { ...components, o: String(order), v: String(read) })
-
-			const canonical = {
-				raw,
-				components,
-				country: "NO",
-				locale: "nb-NO",
-				source: "synth-no-street-led",
-				source_id,
-				corpus_version: "0.10.0",
-				license:
-					"Synthetic — no-street-led; (street, number, postcode, city) from OpenAddresses NO (per-source attribution in the model card)",
-			}
-
-			if (alignAndWrite(write, canonical, "no-street-led")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		return { read, emitted, skipped, contaminated }
-	},
-}

package/lib/recipes/no-street.ts

@@ -1,91 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `no-street` slice recipe — synthetic NO-street counter-example rows: tuples →
- *   {@link synthesizeNoStreetRow} → aligned LabeledRow. The corpus-side counterweight to the
- *   synth-street slice that drove v0.6.1's `dependent_locality` regression — venue+admin and
- *   admin-only rows with explicit absence of any street-side component. Ported from
- *   scripts/build-no-street-slice.mjs.
- */
-
-import { makeLcg } from "@mailwoman/core/utils"
-
-import {
-	alignAndWrite,
-	readTuples,
-	sliceSourceID,
-	SYNTHETIC_TUPLE_LICENSE as LICENSE,
-	type CorpusRecipe,
-} from "#recipes/scaffold"
-import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "#synthesizers/no-street"
-
-/**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const noStreetRecipe: CorpusRecipe = {
-	name: "no-street",
-	description: "No-street counter-example rows: tuples → synthesizeNoStreetRow → aligned LabeledRow",
-	mode: "tuples",
-	async run(opts, write) {
-		if (!opts.input) throw new Error("no-street recipe requires --input <tuples.jsonl>")
-		// Legacy build-no-street-slice.mjs seeded the LCG via makeRandom(opts.seed) (s = seed).
-		const random = makeLcg(opts.seed)
-		const source = opts.sourceName ?? "synth-no-street"
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-
-		for await (const tuple of readTuples(opts.input)) {
-			read++
-
-			if (!tuple.locality || !tuple.region || !tuple.postcode || !tuple.country) {
-				skipped++
-
-				continue
-			}
-
-			for (let v = 0; v < opts.variants; v++) {
-				const synth = synthesizeNoStreetRow(tuple as NoStreetBaseTuple, { random })
-
-				if (!synth) {
-					skipped++
-
-					continue
-				}
-
-				const ok = alignAndWrite(
-					write,
-					{
-						raw: synth.raw,
-						components: synth.components,
-						country: tuple.country,
-						locale: synth.locale,
-						source,
-						source_id: sliceSourceID(source, {
-							locality: tuple.locality,
-							region: tuple.region,
-							postcode: tuple.postcode,
-							country: tuple.country,
-							template: synth.template,
-							v: String(v),
-						}),
-						corpus_version: "0.4.0",
-						license: LICENSE,
-					},
-					synth.template
-				)
-
-				if (ok) {
-					emitted++
-				} else {
-					skipped++
-				}
-			}
-		}
-
-		return { read, emitted, skipped }
-	},
-}