@mailwoman/corpus 7.2.0 → 7.3.0

package/src/shard-recipes/po-box.ts

@@ -0,0 +1,111 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `po-box` shard recipe — synthetic PO box rows: tuples → {@link synthesizePoBoxRow} → aligned
+ *   LabeledRow, plus optional self-contained US military/diplomatic rows (#517) at
+ *   `--military-ratio`. Region is required EXCEPT region-less locales (NZ). Ported from
+ *   scripts/build-po-box-shard.mjs.
+ */
+
+import { synthesizeMilitaryPoBoxRow, synthesizePoBoxRow, type PoBoxBaseTuple } from "../synthesize-po-box.ts"
+import { alignAndWrite, makeLcg, readTuples, shardSourceID, type ShardRecipe } from "./scaffold.ts"
+
+const LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
+
+export const poBoxRecipe: ShardRecipe = {
+	name: "po-box",
+	description: "PO box rows: tuples → synthesizePoBoxRow (+ optional US military/diplomatic rows)",
+	mode: "tuples",
+	options: [
+		{ flag: "--pmb-ratio <p>", description: "P(private-mailbox layout). Default 0.15" },
+		{ flag: "--military-ratio <p>", description: "P(emit one US military/diplomatic row per input, #517). Default 0" },
+	],
+	async run(opts, write) {
+		if (!opts.input) throw new Error("po-box recipe requires --input <tuples.jsonl>")
+		const random = makeLcg(opts.seed)
+		const pmbRatio = opts.pmbRatio ?? 0.15
+		const militaryRatio = opts.militaryRatio ?? 0
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			read++
+			// Region required EXCEPT region-less locales (NZ: "Private Bag 12, Auckland 1010", #517).
+			const regionOptional = ["NZ", "NZL", "NEW ZEALAND"].includes(String(tuple.country || "").toUpperCase())
+
+			if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && !regionOptional)) {
+				skipped++
+				continue
+			}
+
+			for (let v = 0; v < opts.variants; v++) {
+				const synth = synthesizePoBoxRow(tuple as PoBoxBaseTuple, { random, pmbRatio })
+
+				if (!synth) continue
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: synth.raw,
+						components: synth.components,
+						country: tuple.country,
+						locale: synth.locale,
+						source: "synth-po-box",
+						source_id: shardSourceID("synth-po-box", {
+							locality: tuple.locality,
+							region: tuple.region,
+							postcode: tuple.postcode,
+							country: tuple.country,
+							v: String(v),
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					synth.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+
+			// US military/diplomatic rows (#517): self-contained, one per input line at --military-ratio.
+			// Default 0 → byte-stable (random() not called when off). US-only.
+			if (militaryRatio > 0 && random() < militaryRatio) {
+				const mil = synthesizeMilitaryPoBoxRow({ random })
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: mil.raw,
+						components: mil.components,
+						country: "US",
+						locale: mil.locale,
+						source: "synth-po-box",
+						source_id: shardSourceID("synth-po-box", {
+							po_box: mil.components.po_box,
+							locality: mil.components.locality,
+							region: mil.components.region,
+							postcode: mil.components.postcode,
+							v: `mil${emitted}`,
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					mil.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}

package/src/shard-recipes/scaffold.ts

@@ -0,0 +1,187 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Shared scaffolding for the synthetic-corpus SHARD RECIPES — the common bits the 16
+ *   `build-*-shard.mjs` scripts each re-implemented: the seeded LCG PRNG, the tuple reader, and the
+ *   canonical → `alignRow` → `LabeledRow` JSONL emit step. A recipe ({@link ShardRecipe}) supplies
+ *   only its synthesis + filter; the `mailwoman corpus shard <recipe>` command supplies the I/O.
+ */
+
+import { stableSourceID } from "../adapter.ts"
+import { alignRow } from "../align.ts"
+
+/**
+ * {@link stableSourceID}, but accepting arbitrary disambiguator keys (e.g. a variant index `v`) that aren't
+ * `ComponentTag`s. `stableSourceID` sorts + hashes EVERY key it's given, so passing extra keys is how the legacy
+ * builders kept per-variant ids unique — the strict typing is just too narrow for that. Centralizes the one cast.
+ */
+export function shardSourceID(adapterID: string, parts: Record<string, string | undefined>): string {
+	return stableSourceID(adapterID, parts as unknown as Parameters<typeof stableSourceID>[1])
+}
+
+/** A (locality, region, postcode, country) source tuple — the input to tuples-mode recipes. */
+export interface ShardTuple {
+	locality?: string
+	region?: string
+	postcode?: string
+	country?: string
+	[k: string]: unknown
+}
+
+/**
+ * The deterministic LCG (`s = s*1664525 + 1013904223 mod 2^32`) the street/po-box/anchor builders used. A recipe whose
+ * legacy `.mjs` seeded this must create it here so `--seed N` is byte-reproducible.
+ */
+export function makeLcg(seed: number): () => number {
+	let s = seed >>> 0
+
+	return () => {
+		s = (s * 1664525 + 1013904223) % 4294967296
+
+		return s / 4294967296
+	}
+}
+
+/** Back-compat alias for {@link makeLcg}. */
+export const makeRandom = makeLcg
+
+/**
+ * Mulberry32 — the PRNG the MAJORITY of the legacy `build-*-shard` scripts used (german, locale, boundary-stress, unit,
+ * fr-order, country-balanced, intersection, fr-admin-split, street-affix, street-bare, po-box-cedex). A recipe must
+ * seed it EXACTLY as its `.mjs` did (usually `seed`, but some derive a per-stream seed) to stay byte-reproducible.
+ */
+export function makeMulberry32(seed: number): () => number {
+	let a = seed >>> 0
+
+	return () => {
+		a = (a + 0x6d2b79f5) | 0
+		let t = Math.imul(a ^ (a >>> 15), 1 | a)
+		t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t
+
+		return ((t ^ (t >>> 14)) >>> 0) / 4294967296
+	}
+}
+
+/** Stream-parse a tuples JSONL file, yielding each parsed object (blank/invalid lines skipped). */
+export async function* readTuples(input: string): AsyncGenerator<ShardTuple> {
+	// TextSpliterator (not JSONSpliterator) so a malformed line is SKIPPED, not thrown — the
+	// per-line try/catch below is the tolerance this reader has always had.
+	const { TextSpliterator } = await import("spliterator")
+
+	for await (const line of TextSpliterator.fromAsync(input)) {
+		const trimmed = line.trim()
+
+		if (!trimmed) continue
+
+		try {
+			yield JSON.parse(trimmed) as ShardTuple
+		} catch {
+			// skip malformed line
+		}
+	}
+}
+
+/** A canonical row as the recipes assemble it, before `alignRow` turns it into a `LabeledRow`. */
+export interface CanonicalShardRow {
+	raw: string
+	components: Record<string, string>
+	country: string
+	locale?: string
+	source: string
+	source_id: string
+	corpus_version?: string
+	license?: string
+}
+
+/**
+ * Run a canonical row through `alignRow` and, on success, write the `LabeledRow` (+ `synth_method` / `synth_base_id`)
+ * as one JSONL line. Returns true if emitted, false if alignment quarantined it.
+ */
+export function alignAndWrite(
+	write: (line: string) => void,
+	canonical: CanonicalShardRow,
+	synthMethod: string,
+	synthBaseID: string | null = null
+): boolean {
+	const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
+
+	if (!aligned.row) return false
+	write(JSON.stringify({ ...aligned.row, synth_method: synthMethod, synth_base_id: synthBaseID }) + "\n")
+
+	return true
+}
+
+/** Parsed options a recipe's `run` receives. Common fields + the union of recipe-specific flags. */
+export interface ShardRecipeOpts {
+	output: string
+	seed: number
+	variants: number
+	input?: string
+	count?: number
+	golden?: boolean
+	sourceName?: string
+	// recipe-specific (each recipe reads only what it needs):
+	houseNumberProb?: number
+	pmbRatio?: number
+	militaryRatio?: number
+	reversedFraction?: number
+	edgesDir?: string
+	country?: string
+	intlFraction?: number
+	bareProb?: number
+	hnProb?: number
+	communes?: string
+	multilocaleCount?: number
+	/**
+	 * `fr-fragment` / `no-fragment` / `no-street-led`: the eval board's reserved street-surface list. REQUIRED for those
+	 * recipes — a shard that trains on its own eval set measures memorization. See their docstrings.
+	 */
+	excludeSurfaces?: string
+	/** `no-fragment`: share of rows that are counter-distribution (bare locality OR bare postcode). */
+	counterProb?: number
+	/**
+	 * `no-fragment` knob 3: emit N copies of each street+number row whose number has >= longNumberMinDigits digits
+	 * (oversample the failing long-number class). Default 1 = no boost.
+	 */
+	longNumberBoost?: number
+	/** `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted. Default 3. */
+	longNumberMinDigits?: number
+}
+
+/** Tally a recipe returns. */
+export interface ShardStats {
+	read?: number
+	emitted: number
+	skipped: number
+	/**
+	 * Rows dropped because their street SURFACE is reserved by an eval board (`--exclude-surfaces`). Separate from
+	 * `skipped` on purpose: a nonzero value is the audit trail that the train/eval split actually fired. Zero when a
+	 * recipe has no board split.
+	 */
+	contaminated?: number
+}
+
+/** A single declared recipe-specific option flag (for the command's --help). */
+export interface ShardRecipeOption {
+	flag: string
+	description: string
+}
+
+/** A shard recipe: its identity, input mode, and its synthesis `run`. */
+export interface ShardRecipe {
+	/** Recipe id, e.g. "street", "po-box" — the `<recipe>` positional. */
+	name: string
+	/** One-line description for `--list` / help. */
+	description: string
+	/** `tuples` reads `--input` JSONL; `generate` self-generates `--count` rows. */
+	mode: "tuples" | "generate"
+	/** Recipe-specific flags this recipe honors (documentation only). */
+	options?: ShardRecipeOption[]
+	/**
+	 * Do the build: create the recipe's PRNG from `opts.seed` (its LEGACY generator — `makeLcg` or `makeMulberry32` — for
+	 * byte-reproducibility), synthesize, and emit each row via `write`.
+	 */
+	run(opts: ShardRecipeOpts, write: (line: string) => void): Promise<ShardStats>
+}

package/src/shard-recipes/si-bare-village.ts

@@ -0,0 +1,88 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `si-bare-village` — the Slovenian no-street counter-shard (#901 run-2). Slovenia's rural
+ *   addressing has NO street line: the village name is the street-level token and repeats as the
+ *   locality ("Zabiče 8, 6250 Zabiče"). The v1.9.8 gate FALSIFIED the fr-bare-street shard alone on
+ *   exactly this class (SI resolve −3.4pp; "Apače 108" split into street "Apače 10" + house "8") —
+ *   the bare-street boundary lesson generalizes onto a form where the leading name must keep its
+ *   number whole and the trailing mention must stay locality-bound. This shard is the paired
+ *   counter-distribution: same lesson ("name before number, comma, then admin"), opposite polarity
+ *   on the trailing mention.
+ *
+ *   Three real-order templates cycle per tuple (mirrors the coord-golden orders so the eval and the
+ *   training distribution agree):
+ *
+ *   1. canonical  "«V» «n», «pc» «V»"
+ *   2. bare       "«V» «n», «V»"        (no postcode — the anchor-free form, the fr-bare lesson)
+ *   3. pc-first   "«pc» «V», «V» «n»"
+ *
+ *   Gold spans: leading «V» = street (matches OA ground truth — the village IS the address line),
+ *   «n» = house_number (NEVER split mid-digits), «pc» = postcode (never swallowing the neighbor),
+ *   trailing «V» = locality (the binding the resolver needs).
+ */
+
+import { alignAndWrite, makeMulberry32, readTuples, type ShardRecipe, shardSourceID } from "./scaffold.ts"
+
+export const siBareVillageRecipe: ShardRecipe = {
+	name: "si-bare-village",
+	description: "SI no-street village form (#901 run-2): '«V» «n», «pc» «V»' — the fr-bare-street counter-distribution",
+	mode: "tuples",
+	async run(opts, write) {
+		makeMulberry32(opts.seed)
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const t of readTuples(opts.input!)) {
+			read++
+			const village = String(t.locality ?? "").trim()
+			const number = String(t.number ?? "").trim()
+			const postcode = String(t.postcode ?? "").trim()
+
+			if (!village || !number || !postcode) {
+				skipped++
+				continue
+			}
+			const order = read % 3
+			let raw: string
+			const components: Record<string, string> = {
+				street: village,
+				house_number: number,
+				locality: village,
+			}
+
+			if (order === 0) {
+				components.postcode = postcode
+				raw = `${village} ${number}, ${postcode} ${village}`
+			} else if (order === 1) {
+				raw = `${village} ${number}, ${village}`
+			} else {
+				components.postcode = postcode
+				raw = `${postcode} ${village}, ${village} ${number}`
+			}
+			const source_id = shardSourceID("synth-si-bare-village", { ...components, o: String(order), v: String(read) })
+			const canonical = {
+				raw,
+				components,
+				country: "SI",
+				locale: "sl-SI",
+				source: "synth-si-bare-village",
+				source_id,
+				corpus_version: "0.9.9",
+				license:
+					"Synthetic — si-bare-village; (village, number, postcode) from OpenAddresses SI (per-source attribution in the model card)",
+			}
+
+			if (alignAndWrite(write, canonical, "si-bare-village")) {
+				emitted++
+			} else {
+				skipped++
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}