@mailwoman/corpus 10.0.0 → 10.2.0

package/lib/no/recipes/fragment.ts

@@ -2,47 +2,28 @@
  * @copyright Sister Software
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
- *
- *   `no-fragment` — the Norwegian house-number-licence change (Track B, 2026-07-16). The mirror of
- *   `fr-fragment`, which earned +50pp on the same defect shape in French.
- *
- *   WHY THIS EXISTS AND `no-street-led` DOES NOT SUFFICE. Board 3 (the NO digit board) measured that
- *   `synth-no-street-led`'s three forms — all carrying postcode+city — are ALREADY at 0.940-0.968 on
- *   a model with zero Norwegian rows. The headroom is in the forms that slice never emits:
- *
- *     bare-street-hn   "Hallingrudveien 32"     0.693   — no postcode competing, still fails 31%
- *     slash-hn         "Øvrabø 124/1"           0.650   — cadastral gnr/bnr, ONE component
- *
- *   THE MECHANISM the board exposed: `Hallingrudveien 32` -> locality + postcode, while
- *   `Hallingrudveien 32, 3370 Vikersund` parses perfectly. The street loses its street reading and
- *   the digit loses its anchor TOGETHER. That is Track A's bare-street LICENCE in Norwegian — the
- *   model will not read a street without its postcode/locality partner — not a digit-ownership prior.
- *   fr-fragment fixed exactly this in French by teaching the street WITHOUT its partners.
- *
- *   THE COUNTER-DISTRIBUTION IS THE POINT (fr-fragment's lesson, and board 2's bare-locality guard).
- *   Teaching bare `{street} {number}` alone lets the model satisfy every row by flipping its default
- *   from "bare toponym -> locality" to "bare toponym -> street", trading one broken prior for
- *   another. Two counter-classes hold the line:
- *     - bare LOCALITIES (no street) so "bare -> street" is not free.
- *     - bare POSTCODES so the model does not learn to stop emitting postcode to win the digit — the
- *       board 3 bare-pc negative class (1.000) must HOLD.
- *
- *   SLASH HAZARD, pinned deliberately: NO `124/1` is ONE house_number (cadastral gnr/bnr). AU
- *   `12/345` is TWO (unit + house_number). This slice teaches the Norwegian reading; a future AU
- *   intra-word-split slice (B5) must not generalize over it. The two are locale-restricted by design.
- *
- *   SPLIT: `--exclude-surfaces` REQUIRED (throws otherwise) — the digit board's reserved surface
- *   list. Diacritic-KEEPING normalizer, matching the board (see the norm docstring).
  */
 
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
-import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import {
+	alignAndWrite,
+	foldNOSurface,
+	readTuples,
+	requireRegister,
+	type CorpusRecipe,
+	recipeSourceID,
+} from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 
 /**
- * Title-case a Kartverket ALL-CAPS locality (HELLVIK -> Hellvik); #690, all-caps is OOD.
+ * Resolved once, because `source_id` includes the source as its prefix and a pair
+ * that disagreed would name a source no row of this output declares.
  */
+const NO_FRAGMENT_SOURCE = defaultRecipeSource("synth-no-fragment")
+
 const titleNO = (value: string): string =>
 	value
 		.split(/\s+/)
@@ -50,13 +31,12 @@ const titleNO = (value: string): string =>
 		.join(" ")
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * Norwegian fragment recipe.
  */
 export const noFragmentRecipe: CorpusRecipe = {
 	name: "no-fragment",
 	description:
-		"NO street fragments — the house-number-licence change (Track B): '«st» «n»' / bare «st» with NO postcode partner, + bare-locality & bare-postcode counters",
+		"NO street fragments — the house-number-license change (Track B): '«st» «n»' / bare «st» with NO postcode partner, + bare-locality & bare-postcode counters",
 	mode: "tuples",
 	options: [
 		{
@@ -66,7 +46,8 @@ export const noFragmentRecipe: CorpusRecipe = {
 		},
 		{
 			flag: "--bare-street-prob <n>",
-			description: "Share of street rows emitted as a BARE street, no number (default 0.30 — the pure licence signal)",
+			description:
+				"Share of street rows emitted as a bare street without a house number, the pure license signal (default 0.30)",
 		},
 		{
 			flag: "--counter-prob <n>",
@@ -89,7 +70,7 @@ export const noFragmentRecipe: CorpusRecipe = {
 		if (!excludePath) {
 			throw new Error(
 				"no-fragment: --exclude-surfaces is REQUIRED. Pass the NO digit board's reserved list " +
-					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this slice trains on its own eval set. " +
+					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this recipe trains on its own eval set. " +
 					"Source-disjoint by street SURFACE is the split discipline; there is no safe default."
 			)
 		}
@@ -97,10 +78,8 @@ export const noFragmentRecipe: CorpusRecipe = {
 		const excluded = new Set<string>()
 
 		for await (const line of TextSpliterator.fromAsync(excludePath)) {
-			const trimmed = line.trim()
-
-			if (trimmed && !trimmed.startsWith("#")) {
-				excluded.add(trimmed)
+			if (!line.startsWith("#")) {
+				excluded.add(line)
 			}
 		}
 
@@ -111,8 +90,6 @@ export const noFragmentRecipe: CorpusRecipe = {
 		const longNumberBoost = Math.max(1, Math.floor(opts.longNumberBoost ?? 1))
 		const longNumberMinDigits = opts.longNumberMinDigits ?? 3
 
-		// Harvested from the tuples — every NO row carries its locality and postcode, so the two
-		// counter-classes need no second source.
 		const localities = new Set<string>()
 		const postcodes = new Set<string>()
 
@@ -123,28 +100,35 @@ export const noFragmentRecipe: CorpusRecipe = {
 		let emitSeq = 0
 
 		const emit = (raw: string, components: Record<string, string>, klass: string): void => {
-			// emitSeq keeps every emit distinct — knob 3 emits N copies of one long-number row, and
-			// (components, read) alone would collide their source_id and let downstream dedup drop the boost.
-			const source_id = sliceSourceID("synth-no-fragment", { ...components, k: klass, v: `${read}:${emitSeq++}` })
+			const source_id = recipeSourceID(NO_FRAGMENT_SOURCE, { ...components, k: klass, v: `${read}:${emitSeq++}` })
 
 			const canonical = {
 				raw,
 				components,
 				country: "NO",
 				locale: "nb-NO",
-				source: "synth-no-fragment",
+				source: NO_FRAGMENT_SOURCE,
 				source_id,
 				corpus_version: "0.11.0",
 				license: "Synthetic — no-fragment; (street, number, postcode, city) from OpenAddresses NO / Kartverket",
 			}
 
-			if (alignAndWrite(write, canonical, "no-fragment")) {
+			if (alignAndWrite(write, canonical, "no-fragment", NO_FRAGMENT_PROVENANCE)) {
 				emitted++
 			} else {
 				skipped++
 			}
 		}
 
+		const NO_FRAGMENT_PROVENANCE = {
+			register: requireRegister(opts, "no-fragment"),
+			surface: SurfaceOrigin.Composed,
+			// `emit` is a closure over the run rather than over one tuple, so this cannot name a per-row record.
+			// The tuples file has no `sourceID` either.
+			// Both inputs are required to populate this field (#2359).
+			baseSourceID: null,
+		}
+
 		for await (const tuple of readTuples(opts.input!)) {
 			read++
 			const street = String(tuple.street ?? "").trim()
@@ -166,16 +150,12 @@ export const noFragmentRecipe: CorpusRecipe = {
 				continue
 			}
 
-			// THE SPLIT. A surface on the digit board never enters training.
 			if (excluded.has(foldNOSurface(street))) {
 				contaminated++
 
 				continue
 			}
 
-			// COUNTER-DISTRIBUTION — drawn from the harvested pools, not this row's street. Half bare
-			// localities (so "bare -> street" is not free), half bare postcodes (so the model does not
-			// stop emitting postcode to win the digit — board 3's bare-pc must hold).
 			if (random() < counterProb) {
 				if (random() < 0.5 && localities.size) {
 					const loc = [...localities][Math.floor(random() * localities.size)]!
@@ -190,15 +170,11 @@ export const noFragmentRecipe: CorpusRecipe = {
 				continue
 			}
 
-			// THE SIGNAL. A street with NO postcode/locality partner. Either bare, or street+number —
-			// both are the forms board 3 measured as the headroom (bare-street-hn 0.693, slash-hn 0.650).
 			if (!number || random() < bareStreetProb) {
 				emit(street, { street }, "bare-street")
 			} else {
 				const klass = number.includes("/") ? "slash-hn" : "street-hn"
-				// knob 3: the failing class is street + LONG number (Leppdalsvegen 1285 -> postcode). The
-				// digit count, not the slash, is what tips the length prior toward postcode. Oversample
-				// those rows to fight the prior with volume and teach the street/number boundary directly.
+
 				const digits = (number.match(/\d/g) ?? []).length
 				const copies = digits >= longNumberMinDigits ? longNumberBoost : 1
 

package/lib/no/recipes/street/led.ts

@@ -3,27 +3,33 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `no-street-led` — the Norwegian street-led slice, third orthography of the #901
- *   leading-name-boundary family. The post-#920 NO row-read measured the residual as
- *   ORDER-SENSITIVE decode: street-led forms ("Tangavegen 40, 5620 Tørvikbygd") carry a 30%
- *   residual vs pc-first's 7% — the diacritic street head (…vegen/…veien with ø/å/æ) is the same
- *   leading-name-before-comma boundary the FR bare-street and SI village forms exercise, in a
- *   third orthography. All three real orders cycle per tuple (balanced polarity — the v1.9.9
- *   lesson: a one-order gradient loses at convergence):
+ * `no-street-led`: the Norwegian street-led recipe.
  *
- *   1. canonical  "«st» «n», «pc» «city»"   (the 30% residual class — the lesson)
- *   2. city-first "«city», «pc», «st» «n»"
- *   3. pc-first   "«pc» «city», «st» «n»"   (the 7% floor — the anchor the others converge to)
+ * Street-led forms ("Tangavegen 40, 5620 Tørvikbygd") have a larger residual than the pc-first form.
+ * A diacritic street head (…vegen/…veien with ø/å/æ) creates a leading-name-before-comma boundary.
+ * The recipe cycles each tuple through all three real orders to balance polarity.
+ *
+ * 1. canonical  "«st» «n», «pc» «city»"   (the larger residual class)
+ * 2. city-first "«city», «pc», «st» «n»"
+ * 3. pc-first   "«pc» «city», «st» «n»"   (the floor the others converge to)
  */
 
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 import { TextSpliterator } from "spliterator"
 
-import { alignAndWrite, foldNOSurface, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import {
+	alignAndWrite,
+	foldNOSurface,
+	readTuples,
+	requireRegister,
+	type CorpusRecipe,
+	recipeSourceID,
+} from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * Recipe registered with the corpus builder.
  */
 export const noStreetLedRecipe: CorpusRecipe = {
 	name: "no-street-led",
@@ -40,16 +46,15 @@ export const noStreetLedRecipe: CorpusRecipe = {
 	async run(opts, write) {
 		makeMulberry32(opts.seed)
 
-		// THE SPLIT (ported from fr-fragment, #727 T2). Without it this slice trains on all 10,697
-		// NO surfaces, 1,952 of which the digit board reserves — so a Norway retrain would grade
-		// memorization of `Hallingrudveien` while claiming to measure the boundary form. There is no
-		// safe default: source-disjoint by street SURFACE is the discipline, so the flag throws.
+		// The split excludes every surface the digit board reserves, so a retrain
+		// grades the boundary form on unseen streets.
+		// There is no safe default, so the flag throws.
 		const excludePath = opts.excludeSurfaces
 
 		if (!excludePath) {
 			throw new Error(
 				"no-street-led: --exclude-surfaces is REQUIRED. Pass the NO digit board's reserved list " +
-					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this slice trains on its own eval set. " +
+					"(mailwoman/eval-harness/fixtures/no-digits.surfaces.txt) or this recipe trains on its own eval set. " +
 					"Source-disjoint by street SURFACE is the split discipline; there is no safe default."
 			)
 		}
@@ -57,10 +62,8 @@ export const noStreetLedRecipe: CorpusRecipe = {
 		const excluded = new Set<string>()
 
 		for await (const line of TextSpliterator.fromAsync(excludePath)) {
-			const trimmed = line.trim()
-
-			if (trimmed && !trimmed.startsWith("#")) {
-				excluded.add(trimmed)
+			if (!line.startsWith("#")) {
+				excluded.add(line)
 			}
 		}
 
@@ -71,6 +74,11 @@ export const noStreetLedRecipe: CorpusRecipe = {
 		let skipped = 0
 		let contaminated = 0
 
+		const NO_STREET_LED_PROVENANCE = {
+			register: requireRegister(opts, "no-street-led"),
+			surface: SurfaceOrigin.Composed,
+		}
+
 		for await (const t of readTuples(opts.input!)) {
 			read++
 			const street = String(t.street ?? "").trim()
@@ -109,21 +117,32 @@ export const noStreetLedRecipe: CorpusRecipe = {
 				raw = `${postcode} ${city}, ${street} ${number}`
 			}
 
-			const source_id = sliceSourceID("synth-no-street-led", { ...components, o: String(order), v: String(read) })
+			// One resolution for both, because `source_id` includes the source as its prefix
+			// and a pair that disagreed would name a source no row of this output declares.
+			const source = defaultRecipeSource("synth-no-street-led")
+			const source_id = recipeSourceID(source, { ...components, o: String(order), v: String(read) })
 
 			const canonical = {
 				raw,
 				components,
 				country: "NO",
 				locale: "nb-NO",
-				source: "synth-no-street-led",
+				source,
 				source_id,
 				corpus_version: "0.10.0",
 				license:
 					"Synthetic — no-street-led; (street, number, postcode, city) from OpenAddresses NO (per-source attribution in the model card)",
 			}
 
-			if (alignAndWrite(write, canonical, "no-street-led")) {
+			// Per row, because the id names this tuple's record.
+			// Tuple files created before `sourceID` have no identifier to forward.
+			// A `null` base ID preserves that absence.
+			if (
+				alignAndWrite(write, canonical, "no-street-led", {
+					...NO_STREET_LED_PROVENANCE,
+					baseSourceID: t.sourceID ?? null,
+				})
+			) {
 				emitted++
 			} else {
 				skipped++

package/lib/no/recipes/{street/index.ts → street.ts}

@@ -3,11 +3,10 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `no-street` slice recipe — synthetic NO-street counter-example rows: tuples →
+ *   `no-street` recipe — synthetic no-street counter-example rows: tuples →
  *   {@link synthesizeNoStreetRow} → aligned LabeledRow. The corpus-side counterweight to the
- *   synth-street slice that drove v0.6.1's `dependent_locality` regression — venue+admin and
- *   admin-only rows with explicit absence of any street-side component. Ported from
- *   scripts/build-no-street-slice.mjs.
+ *   `invented-street` source — venue+admin and admin-only rows with explicit absence of any
+ *   street-side component.
  */
 
 import { makeLcg } from "@mailwoman/core/utils"
@@ -15,15 +14,33 @@ import { makeLcg } from "@mailwoman/core/utils"
 import {
 	alignAndWrite,
 	readTuples,
-	sliceSourceID,
+	recipeSourceID,
 	SYNTHETIC_TUPLE_LICENSE as LICENSE,
 	type CorpusRecipe,
 } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
 import { synthesizeNoStreetRow, type NoStreetBaseTuple } from "#synthesizers/no-street"
+import { SurfaceOrigin } from "#types"
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * A venue or admin-only row asserting that no street-side component is present.
+ *
+ * The row uses components from the `--input` tuples.
+ * It teaches the absence of a street component instead of representing a source record.
+ * No register asserts the row.
+ */
+const NO_STREET_PROVENANCE = {
+	register: null,
+	surface: SurfaceOrigin.Invented,
+	// An invented row has no underlying record to name.
+	baseSourceID: null,
+}
+
+/**
+ * Recipe registered with the corpus builder.
+ *
+ * The file header describes the parse behavior this recipe exercises.
+ * `description` below shows the generated surface form.
  */
 export const noStreetRecipe: CorpusRecipe = {
 	name: "no-street",
@@ -31,9 +48,9 @@ export const noStreetRecipe: CorpusRecipe = {
 	mode: "tuples",
 	async run(opts, write) {
 		if (!opts.input) throw new Error("no-street recipe requires --input <tuples.jsonl>")
-		// Legacy build-no-street-slice.mjs seeded the LCG via makeRandom(opts.seed) (s = seed).
+		// The legacy build script seeded the LCG via makeRandom(opts.seed) (s = seed).
 		const random = makeLcg(opts.seed)
-		const source = opts.sourceName ?? "synth-no-street"
+		const source = opts.sourceName ?? defaultRecipeSource("synth-no-street")
 		let read = 0
 		let emitted = 0
 		let skipped = 0
@@ -64,7 +81,7 @@ export const noStreetRecipe: CorpusRecipe = {
 						country: tuple.country,
 						locale: synth.locale,
 						source,
-						source_id: sliceSourceID(source, {
+						source_id: recipeSourceID(source, {
 							locality: tuple.locality,
 							region: tuple.region,
 							postcode: tuple.postcode,
@@ -75,7 +92,8 @@ export const noStreetRecipe: CorpusRecipe = {
 						corpus_version: "0.4.0",
 						license: LICENSE,
 					},
-					synth.template
+					synth.template,
+					NO_STREET_PROVENANCE
 				)
 
 				if (ok) {