@mailwoman/corpus 9.2.0 → 9.4.0

package/src/shard-recipes/anchor-absorption.ts

@@ -1,103 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `anchor-absorption` shard recipe (#220/#723, Probe A1) — self-generates `--count` rows from
- *   {@link synthesizeAnchorAbsorptionRow}'s 6-slice mix (CASE-H / CASE-P-us-rural / CASE-P-de /
- *   anchor-fp / locale-ambig / standard), aligns each to BIO, and emits a labeled JSONL. The
- *   LEADING 5-digit on CASE-H/anchor-fp/locale-ambig is sampled from the REAL US ZIPs in the
- *   postcode-anchor lookup, so the shaped-painted anchor fires on it exactly as inference does —
- *   teaching the model to OVERRIDE a present anchor from context. Ported from
- *   scripts/build-anchor-absorption-shard.mjs.
- */
-
-import { readFileSync } from "node:fs"
-
-import { parseJSONStrict } from "@mailwoman/core/objects"
-import { dataRootPath } from "@mailwoman/core/utils"
-
-import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption"
-import { alignRow } from "#utils"
-
-import { makeLcg, shardSourceID, type ShardRecipe } from "./scaffold.ts"
-
-/**
- * The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
- * helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
- */
-const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json")
-
-/**
- * The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list).
- */
-function loadRealUsZips(path: string): string[] {
-	const d = parseJSONStrict<Record<string, unknown>>(readFileSync(path, "utf8"))
-	const zips: string[] = []
-
-	for (const [pc, v] of Object.entries(d)) {
-		if (Array.isArray(v) && v[0] && typeof v[0] === "object" && "US" in (v[0] as object) && /^\d{5}$/.test(pc)) {
-			zips.push(pc)
-		}
-	}
-
-	return zips
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const anchorAbsorptionRecipe: ShardRecipe = {
-	name: "anchor-absorption",
-	description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
-	mode: "generate",
-	async run(opts, write) {
-		// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
-		const random = makeLcg(opts.seed)
-		const source = opts.sourceName ?? "synth-anchor-absorption"
-		const count = opts.count ?? 50_000
-		const realZips = loadRealUsZips(ANCHOR_LOOKUP)
-
-		console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`)
-
-		let written = 0
-		let quarantined = 0
-		const byTemplate: Record<string, number> = {}
-
-		for (let i = 0; i < count; i++) {
-			const synth = synthesizeAnchorAbsorptionRow({ random, realZips })
-			const country = synth.locale.split("-")[1]
-
-			// "en-US" -> "US", "de-DE" -> "DE"
-			const canonical = {
-				raw: synth.raw,
-				components: synth.components,
-				country,
-				locale: synth.locale,
-				source,
-				source_id: shardSourceID(source, { v: String(i) }),
-			}
-
-			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
-
-			if (aligned.kind !== "labeled") {
-				quarantined++
-
-				continue
-			}
-
-			write(
-				JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n"
-			)
-
-			written++
-			byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1
-		}
-
-		console.error(`\nwrote ${written} rows (${quarantined} quarantined)`)
-		console.error("  by slice:", JSON.stringify(byTemplate))
-
-		return { emitted: written, skipped: quarantined }
-	},
-}

package/src/shard-recipes/bare-country.ts

@@ -1,95 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
- *   country name, so the class is out of the model's training distribution and the tag is a coin
- *   flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
- *   `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
- *   bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
- *   tag itself is right.
- *
- *   Source is the codex country table — every ISO canonical English name plus the curated surface
- *   forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
- *   `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
- *   English words), and teaching it here would be an unmeasured claim. Case augmentation is the
- *   loader's job, as everywhere.
- */
-
-import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country"
-
-import { alignAndWrite, makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
-
-/**
- * Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
- */
-const MIN_NAME_LENGTH = 4
-
-/**
- * Every (surface, iso2) pair the shard emits: the ISO canonical names plus each country's curated surface forms,
- * deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
- */
-function* bareCountrySurfaces(): Generator<{ surface: string; iso2: string }> {
-	const seen = new Set<string>()
-
-	for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
-		for (const surface of forms) {
-			if (surface.length < MIN_NAME_LENGTH || seen.has(surface)) continue
-			seen.add(surface)
-			yield { surface, iso2 }
-		}
-	}
-
-	for (const name of CountryNames) {
-		if (name.length < MIN_NAME_LENGTH || seen.has(name)) continue
-		const matched = matchCountry(name)
-
-		if (!matched) continue
-		seen.add(name)
-		yield { surface: name, iso2: matched.iso2 }
-	}
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const bareCountryRecipe: ShardRecipe = {
-	name: "bare-country",
-	description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
-	mode: "generate",
-	async run(opts, write) {
-		// Seeded for parity with the other recipes; the surfaces themselves drive the content.
-		makeMulberry32(opts.seed)
-		let read = 0
-		let emitted = 0
-		let skipped = 0
-
-		for (const { surface, iso2 } of bareCountrySurfaces()) {
-			read++
-
-			const components: Record<string, string> = { country: surface }
-			const source_id = shardSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) })
-
-			const canonical = {
-				raw: surface,
-				components,
-				country: iso2,
-				locale: "und",
-				source: "synth-bare-country",
-				source_id,
-				corpus_version: "0.11.0",
-				license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
-			}
-
-			if (alignAndWrite(write, canonical, "bare-country")) {
-				emitted++
-			} else {
-				skipped++
-			}
-		}
-
-		return { read, emitted, skipped }
-	},
-}

package/src/shard-recipes/boundary-stress.ts

@@ -1,101 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `boundary-stress` shard recipe (#375) — the boundary-instability augmentation. Self-generates
- *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
- *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
- *   lever for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
- *   scripts/build-boundary-stress-shard.mjs.
- *
- *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
- */
-
-import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
-import { alignRow } from "#utils"
-
-import { makeMulberry32, type ShardRecipe, shardSourceID } from "./scaffold.ts"
-
-/**
- * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
- * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
- * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
- * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is load-bearing — it drives the cumulative
- * thresholds below.
- */
-const WEIGHTS: Record<BoundaryStressTemplate, number> = {
-	"street-eats-affix": 0.22,
-	"comma-less-city-state": 0.22,
-	"fr-prefix": 0.18,
-	"bare-locality": 0.11,
-	"house-number-before-street": 0.189,
-	"house-number-after-street": 0.081,
-}
-
-const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
-	let acc = 0
-
-	return (Object.entries(WEIGHTS) as Array<[BoundaryStressTemplate, number]>).map(
-		([t, w]) => [t, (acc += w)] as [BoundaryStressTemplate, number]
-	)
-})()
-
-function pickTemplate(r: () => number): BoundaryStressTemplate {
-	const x = r()
-
-	for (const [t, c] of CUM) if (x <= c) return t
-
-	return CUM.at(-1)![0]
-}
-
-/**
- * Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
- */
-export const boundaryStressRecipe: ShardRecipe = {
-	name: "boundary-stress",
-	description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
-	mode: "generate",
-	async run(opts, write) {
-		// Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
-		const random = makeMulberry32(opts.seed)
-		const count = opts.count ?? 20_000
-		let emitted = 0
-		let skipped = 0
-
-		for (let i = 0; i < count; i++) {
-			const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) })
-			const country = row.locale.split("-")[1] ?? "US"
-			const source_id = shardSourceID("synth-boundary-stress", { ...row.components, v: String(i) })
-
-			const canonical = {
-				raw: row.raw,
-				components: row.components,
-				country,
-				locale: row.locale,
-				source: "synth-boundary-stress",
-				source_id,
-				corpus_version: "0.6.0",
-				license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
-			}
-
-			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
-
-			if (aligned.kind !== "labeled") {
-				skipped++
-
-				continue
-			}
-
-			// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
-			write(
-				JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n"
-			)
-
-			emitted++
-		}
-
-		return { emitted, skipped }
-	},
-}