@mailwoman/corpus 9.3.0 → 10.0.0

package/lib/recipes/street/bare.ts

@@ -0,0 +1,102 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `street-bare` slice recipe — BARE-street rows (the v0.8.0 harness change, 2026-06-05). The
+ *   `functional.test.ts` cluster (bare street names — "10th Ave", "Main St", "1 Main Pl") was
+ *   mislabeled `locality` because {@link synthesizeStreetRow} only ever emitted streets WITH a ",
+ *   City, ST ZIP" tail. This recipe emits streets BARE (`--bare-prob`, default 0.6) — no tail, only
+ *   `street_prefix`/`street`/`street_suffix` (+ optional `house_number` at `--hn-prob`, default
+ *   0.85) — over the built-in {@link DEFAULT_US_BASES} pool (round-robin). Generate-mode, US-only,
+ *   in-distribution (no German-collapse risk). Ported from scripts/build-street-bare-slice.mjs.
+ *
+ *   Byte-fidelity: the legacy script seeded its own mulberry32 from `--seed`
+ *   (`mulberry32(opts.seed)`); this recipe re-creates the SAME generator
+ *   (`makeMulberry32(opts.seed)`) and preserves the synthesis call order exactly, so `--seed N`
+ *   reproduces the legacy run byte-for-byte.
+ */
+
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+
+import { sliceSourceID, type CanonicalSliceRow, type CorpusRecipe } from "#recipes/scaffold"
+import { DEFAULT_US_BASES } from "#synthesizers/intersection"
+import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
+import { alignRow } from "#utils"
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const streetBareRecipe: CorpusRecipe = {
+	name: "street-bare",
+	description: "Bare-street rows (US): DEFAULT_US_BASES → synthesizeStreetRow (bare) → aligned LabeledRow",
+	mode: "generate",
+	options: [
+		{ flag: "--bare-prob <p>", description: "P(emit the street BARE — no city/region/postcode tail). Default 0.6" },
+		{ flag: "--hn-prob <p>", description: "P(emit a house number). Default 0.85" },
+	],
+	async run(opts, write) {
+		const random = makeMulberry32(opts.seed)
+		const count = opts.count ?? 3000
+		const bareProb = opts.bareProb ?? 0.6
+		const hnProb = opts.hnProb ?? 0.85
+		const source = opts.sourceName ?? "synth-street-bare"
+
+		let emitted = 0
+		let skipped = 0
+		let guard = 0
+
+		while (emitted < count && guard++ < count * 5) {
+			const base = DEFAULT_US_BASES[emitted % DEFAULT_US_BASES.length]!
+
+			const synth = synthesizeStreetRow(base as StreetBaseTuple, {
+				random,
+				bareProb,
+				includeHouseNumberProb: hnProb,
+			})
+
+			if (!synth) {
+				skipped++
+
+				continue
+			}
+
+			const isBare = synth.components.region === undefined
+
+			const canonical: CanonicalSliceRow = {
+				raw: synth.raw,
+				components: synth.components,
+				country: base.country,
+				locale: synth.locale,
+				source,
+				source_id: sliceSourceID(source, {
+					street: synth.components.street,
+					street_suffix: synth.components.street_suffix,
+					house_number: synth.components.house_number,
+					bare: String(isBare),
+					n: String(emitted),
+				}),
+				corpus_version: "0.4.0",
+				license: "Synthetic — US street templates, public-domain street/city pools",
+			}
+
+			// Strict labeled-only check (matches the legacy builder): alignRow always returns a `row`
+			// (labeled OR quarantined), so the scaffold's `alignAndWrite` would write quarantined rows
+			// too — call alignRow directly and skip anything not "labeled".
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				skipped++
+
+				continue
+			}
+
+			write(JSON.stringify({ ...aligned.row, synth_method: "street-bare", synth_base_id: null }) + "\n")
+
+			emitted++
+		}
+
+		return { emitted, skipped }
+	},
+}

package/lib/recipes/street.ts

@@ -7,8 +7,10 @@
  *   {@link synthesizeStreetRow} → aligned LabeledRow. Ported from scripts/build-street-slice.mjs.
  */
 
+import { makeLcg } from "@mailwoman/core/utils"
+
 import { stableSourceID } from "#adapters/utils"
-import { alignAndWrite, makeLcg, readTuples, type CorpusRecipe } from "#recipes/scaffold"
+import { alignAndWrite, readTuples, type CorpusRecipe } from "#recipes/scaffold"
 import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
 
 /**

package/lib/recipes/sub/venue/context.ts

@@ -0,0 +1,149 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Reads address context and derives street-side negatives for the sub-venue recipe.
+ */
+
+import { dataRootPath } from "@mailwoman/core/data-root"
+import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+import type { PathBuilderLike } from "path-ts"
+
+import { readTuples as readLocaleTuples, type LocalePart } from "#international/recipes/locale"
+import { readTuples as readSliceTuples } from "#recipes/scaffold"
+import type { LocaleBaseTuple } from "#synthesizers/locale"
+
+//#region Address context
+
+/**
+ * DE + ES address context. Both read through {@link readLocaleTuples}, the `locale` recipe's own streaming + reservoir
+ * reader, so the CSV handling (quoted fields, CRLF, city-noise cleaning, the DE per-part region fallback) has exactly
+ * one implementation.
+ *
+ * DE reads `europe.zip`'s two members rather than `oa-cache/de__*.zip`: the cached per-state zips the `locale` recipe
+ * names are not materialized on this host, and the archive members are byte-identical to OA's current run (verified in
+ * `corpus/AGENTS.md`'s "a file's mtime is not its data's vintage" note).
+ */
+const CONTEXT_PARTS: Readonly<Record<string, readonly LocalePart[]>> = {
+	DE: [
+		{ zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/berlin.csv", region: "Berlin" },
+		{ zip: dataRootPath("openaddresses", "europe.zip"), csv: "de/sn/statewide.csv", region: "Sachsen" },
+	],
+	ES: [{ path: dataRootPath("openaddresses", "extracted", "es", "countrywide.csv") }],
+}
+
+/**
+ * Load the address skeletons every leg renders onto: GB / US / FR from the house-venue v3 tuples (the same 176,519 real
+ * rows the `synth-house-venue` slice is built from, so the two slices' address halves are drawn from one pool), DE and
+ * ES streamed out of OpenAddresses.
+ */
+export async function loadContextTuples(
+	tuplesPath: PathBuilderLike,
+	seed: number
+): Promise<Map<string, LocaleBaseTuple[]>> {
+	const byCountry = new Map<string, LocaleBaseTuple[]>()
+
+	for await (const tuple of readSliceTuples(tuplesPath)) {
+		const country = String(tuple.country ?? "")
+
+		if (!country || !tuple.locality || !tuple.street) continue
+
+		const mapped: LocaleBaseTuple = {
+			house_number: String(tuple.houseNumber ?? tuple.house_number ?? ""),
+			street: String(tuple.street),
+			locality: String(tuple.locality),
+			region: String(tuple.region ?? ""),
+			postcode: String(tuple.postcode ?? ""),
+		}
+
+		const list = byCountry.get(country)
+
+		if (list) {
+			list.push(mapped)
+		} else {
+			byCountry.set(country, [mapped])
+		}
+	}
+
+	for (const [country, parts] of Object.entries(CONTEXT_PARTS)) {
+		if (byCountry.has(country)) continue
+		const pooled: LocaleBaseTuple[] = []
+
+		for (const [index, part] of parts.entries()) {
+			// A dedicated stream PRNG, seeded per part, so the input sample is reproducible without
+			// perturbing the emit loop's draws (the `locale` recipe's rule, kept).
+			const streamRandom = makeMulberry32(seed + index)
+
+			for (const tuple of await readLocaleTuples(part, streamRandom)) {
+				pooled.push(tuple)
+			}
+		}
+
+		byCountry.set(country, pooled)
+	}
+
+	return byCountry
+}
+
+/**
+ * The street-side confound classes, mined from the leg's OWN address tuples.
+ *
+ * Real streets, not invented ones. The 176,519-row context pool carries 195 GB `hall` streets, 114 GB `gate` streets,
+ * 134 distinct GB `-gate` single tokens and a two-figure `<modifier> <designator>` population in both GB and US — small
+ * absolute numbers, but every one of them a street somebody lives on, which is the property an invented list cannot
+ * have.
+ */
+export interface StreetNegatives {
+	designator: LocaleBaseTuple[]
+	modifierDesignator: LocaleBaseTuple[]
+	gateSuffix: LocaleBaseTuple[]
+}
+
+/**
+ * Shortest token that can carry a `-gate` street suffix and still be a NAME rather than the bare word: `gate` itself is
+ * four characters, so the class starts at five (`Highgate`, `Moorgate`, `Stonegate`).
+ */
+const MIN_GATE_SUFFIX_TOKEN_LENGTH = 5
+
+export function buildStreetNegatives(
+	context: readonly LocaleBaseTuple[],
+	designatorPhrases: readonly string[],
+	modifiers: readonly string[],
+	country: string
+): StreetNegatives {
+	const designatorSet = new Set(designatorPhrases)
+	const modifierSet = new Set(modifiers)
+	const designator: LocaleBaseTuple[] = []
+	const modifierDesignator: LocaleBaseTuple[] = []
+	const gateSuffix: LocaleBaseTuple[] = []
+
+	for (const tuple of context) {
+		const tokens = tuple.street.toLowerCase().match(/[\p{L}]+/gu) ?? []
+		let isDesignator = false
+		let isPair = false
+
+		for (const [index, token] of tokens.entries()) {
+			if (designatorSet.has(token)) {
+				isDesignator = true
+			}
+
+			if (index + 1 < tokens.length && modifierSet.has(token) && designatorSet.has(tokens[index + 1]!)) {
+				isPair = true
+			}
+
+			if (country === "GB" && token.length >= MIN_GATE_SUFFIX_TOKEN_LENGTH && token.endsWith("gate")) {
+				gateSuffix.push(tuple)
+			}
+		}
+
+		if (isPair) {
+			modifierDesignator.push(tuple)
+		} else if (isDesignator) {
+			designator.push(tuple)
+		}
+	}
+
+	return { designator, modifierDesignator, gateSuffix }
+}
+
+//#endregion