@mailwoman/corpus 9.2.0 → 9.3.0

package/lib/recipes/boundary-stress.ts

@@ -0,0 +1,100 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `boundary-stress` slice recipe (#375) — the boundary-instability augmentation. Self-generates
+ *   `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
+ *   DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
+ *   change for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
+ *   scripts/build-boundary-stress-slice.mjs.
+ *
+ *   `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
+ */
+
+import { makeMulberry32, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import { type BoundaryStressTemplate, synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress"
+import { alignRow } from "#utils"
+
+/**
+ * Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
+ * "City, STATE" rows WITHOUT becoming a locality-first majority), and house-number-before:after = 7:3 (FR's dominant
+ * order is number-BEFORE; 30% after breaks the order-bias shortcut without risking FR hn-before accuracy). The three
+ * original non-number shapes keep the bulk. Weights sum to 1.0. Key order is required — it drives the cumulative
+ * thresholds below.
+ */
+const WEIGHTS: Record<BoundaryStressTemplate, number> = {
+	"street-eats-affix": 0.22,
+	"comma-less-city-state": 0.22,
+	"fr-prefix": 0.18,
+	"bare-locality": 0.11,
+	"house-number-before-street": 0.189,
+	"house-number-after-street": 0.081,
+}
+
+const CUM: Array<[BoundaryStressTemplate, number]> = (() => {
+	let acc = 0
+
+	return (Object.entries(WEIGHTS) as Array<[BoundaryStressTemplate, number]>).map(
+		([t, w]) => [t, (acc += w)] as [BoundaryStressTemplate, number]
+	)
+})()
+
+function pickTemplate(r: () => number): BoundaryStressTemplate {
+	const x = r()
+
+	for (const [t, c] of CUM) if (x <= c) return t
+
+	return CUM.at(-1)![0]
+}
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const boundaryStressRecipe: CorpusRecipe = {
+	name: "boundary-stress",
+	description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
+	mode: "generate",
+	async run(opts, write) {
+		// Emit PRNG: the legacy build-boundary-stress-slice.mjs seeded mulberry32(opts.seed).
+		const random = makeMulberry32(opts.seed)
+		const count = opts.count ?? 20_000
+		let emitted = 0
+		let skipped = 0
+
+		for (let i = 0; i < count; i++) {
+			const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) })
+			const country = row.locale.split("-")[1] ?? "US"
+			const source_id = sliceSourceID("synth-boundary-stress", { ...row.components, v: String(i) })
+
+			const canonical = {
+				raw: row.raw,
+				components: row.components,
+				country,
+				locale: row.locale,
+				source: "synth-boundary-stress",
+				source_id,
+				corpus_version: "0.6.0",
+				license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
+			}
+
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
+
+			if (aligned.kind !== "labeled") {
+				skipped++
+
+				continue
+			}
+
+			// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
+			write(
+				JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n"
+			)
+
+			emitted++
+		}
+
+		return { emitted, skipped }
+	},
+}

package/lib/recipes/cn-organizational-units.ts

@@ -0,0 +1,199 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file `cn-organizational-units` recipe (#2034) — real CN address strings whose settlement is an ORGANIZATIONAL
+ *   ladder (`赵光三分场二十九队`: the Zhaoguang farm, No. 3 sub-farm, No. 29 production team), labeled by rule and aligned
+ *   character by character for the CJK sibling model.
+ *
+ *   THE LABELS ARE A READING OF THE SUFFIX, NOT A GUESS. Every generic that ends an ordinal unit (`分场`, `大队`, `队`,
+ *   `连`, `团`, `组`, `场部`) is in `@mailwoman/core/locale/zh-cn-units`' one table, and the same table reads the span back
+ *   after decode. The whole ordinal chain is ONE `locality_unit` span; the named head it belongs to (`赵光`, `孟定农场`) is
+ *   `dependent_locality`; a province, city or county written in front of it takes `region`, `locality`, `subregion`; a
+ *   Latin admin tail (`, Heilongjiang, China`) takes `region` and `country`. A row with no chain is skipped, not labeled:
+ *   `红卫大队` is a village name whose generic carries no ordinal, and `苗辽林场` is a named forest farm.
+ *
+ *   WHERE THE ROWS COME FROM. `--input` is a JSONL of `{ raw, country }` rows — the shape of `data/coarse-placer/*.jsonl`,
+ *   whose 50,000 CN rows hold 328 with unit vocabulary. That file is a local artifact and carries no per-row source; the
+ *   `<name>, <admin1>, <country>` shape is the corpus's GeoNames adapter's, so the rows are stamped with GeoNames'
+ *   licence and the inference is stated in the `license` field rather than hidden behind it.
+ *
+ *   FOR THE CJK MODEL ONLY. The tokenizer is {@link cjkAwareTokenizer}: one token per Han character, because the
+ *   whitespace tokenizer reads `三分场八队` as one word and could never give it two labels. The Latin model never trains
+ *   on this recipe's rows; its label set has no `locality_unit`.
+ */
+
+import { splitCNUnitChain } from "@mailwoman/core/locale/zh-cn-units"
+
+import { type CorpusRecipe, readTuples, sliceSourceID } from "#recipes/scaffold"
+import { alignRow } from "#utils/align"
+import { cjkAwareTokenizer } from "#utils/tokenize"
+
+/**
+ * The leading run of Han characters (plus the digits an ordinal may be written with): the Chinese half of a row.
+ */
+const LEADING_HAN = /^[\p{Script=Han}〇\d]+/u
+
+/**
+ * Admin prefixes a CJK address writes in front of the settlement, coarsest first. Each is matched at the START of what
+ * remains, so `云南省临沧市孟定农场三分场二队` peels `云南省` (province), `临沧市` (city), and hands `孟定农场三分场二队` to the unit reader.
+ */
+const ADMIN_PREFIXES: ReadonlyArray<readonly [pattern: RegExp, tag: "region" | "locality" | "subregion"]> = [
+	[/^(.+?(?:省|自治区))/u, "region"],
+	[/^(.+?市)/u, "locality"],
+	[/^(.+?(?:自治县|县|旗))/u, "subregion"],
+]
+
+/**
+ * The components a row's string supports, every value a verbatim substring of `raw`, or `null` when the string carries
+ * no organizational chain and so teaches nothing this recipe exists for.
+ */
+export function labelCNOrganizationalRow(raw: string): Record<string, string> | null {
+	const han = LEADING_HAN.exec(raw)?.[0]
+
+	if (!han) return null
+
+	const components: Record<string, string> = {}
+	let rest = han
+
+	for (const [pattern, tag] of ADMIN_PREFIXES) {
+		const match = pattern.exec(rest)
+
+		// A prefix must leave something behind it, or the whole run was the admin name and there is no settlement.
+		if (match && match[1]!.length < rest.length) {
+			components[tag] = match[1]!
+			rest = rest.slice(match[1]!.length)
+		}
+	}
+
+	const split = splitCNUnitChain(rest)
+
+	if (!split) return null
+
+	if (split.head) {
+		components.dependent_locality = split.head
+	}
+
+	components.locality_unit = split.chain
+
+	const tail = raw.slice(han.length).trim()
+
+	if (tail) {
+		// `, Heilongjiang, China` or `Hunan China` or `Inner Mongolia`: comma segments when there are commas, else the last
+		// word is the country when it says so and the rest is the admin1 name.
+		const segments = tail.includes(",")
+			? tail
+					.split(",")
+					.map((segment) => segment.trim())
+					.filter((segment) => segment.length > 0)
+			: tailWithoutCommas(tail)
+
+		for (const segment of segments) {
+			if (/^china$/iu.test(segment)) {
+				components.country = segment
+			} else if (!components.region && !/^\d+$/u.test(segment)) {
+				components.region = segment
+			}
+		}
+	}
+
+	return components
+}
+
+/**
+ * A space-separated tail: `Hunan China` → [`Hunan`, `China`]; `Inner Mongolia` → [`Inner Mongolia`]; `Xinjiang Uyghur`
+ * → [`Xinjiang Uyghur`].
+ */
+function tailWithoutCommas(tail: string): string[] {
+	const words = tail.split(/\s+/u).filter((word) => word.length > 0)
+	const last = words.at(-1)
+
+	if (last && /^china$/iu.test(last) && words.length > 1) {
+		return [words.slice(0, -1).join(" "), last]
+	}
+
+	return [words.join(" ")]
+}
+
+const SOURCE = "coarse-placer-cn-units"
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the rows it labels and why every label is a
+ * reading of a generic rather than a guess.
+ */
+export const cnOrganizationalUnitsRecipe: CorpusRecipe = {
+	name: "cn-organizational-units",
+	description:
+		"CN rows whose settlement is an organizational ladder (分场/队/连/组), labeled by the suffix grammar as one locality_unit span, per-character BIO",
+	mode: "tuples",
+	options: [],
+	async run(opts, write) {
+		if (!opts.input) {
+			throw new Error("cn-organizational-units: --input <rows.jsonl> is required ({ raw, country } rows)")
+		}
+
+		const tokenizer = cjkAwareTokenizer()
+		let emitted = 0
+		let skipped = 0
+		let quarantined = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			const raw = typeof tuple["raw"] === "string" ? tuple["raw"].trim() : ""
+			const country = typeof tuple["country"] === "string" ? tuple["country"].toUpperCase() : "CN"
+
+			if (!raw || country !== "CN") {
+				skipped++
+
+				continue
+			}
+
+			const components = labelCNOrganizationalRow(raw)
+
+			if (!components) {
+				skipped++
+
+				continue
+			}
+
+			if (opts.golden) {
+				write(JSON.stringify({ raw, components, country: "CN", locale: "zh-CN" }) + "\n")
+
+				emitted++
+
+				continue
+			}
+
+			const canonical = {
+				raw,
+				components,
+				country: "CN",
+				locale: "zh-CN",
+				source: opts.sourceName ?? SOURCE,
+				source_id: sliceSourceID(opts.sourceName ?? SOURCE, { raw }),
+				corpus_version: "0.4.0",
+				license:
+					"CC-BY-4.0 — GeoNames populated places, INFERRED from the `<name>, <admin1>, <country>` row shape; data/coarse-placer carries no per-row source",
+			}
+
+			// Verbatim only: every value above was sliced out of `raw`, so an edit-distance match would mean this file
+			// has a bug, not that the source spells something differently.
+			const aligned = alignRow(canonical as Parameters<typeof alignRow>[0], { tokenizer, maxEditDistance: 0 })
+
+			if (aligned.kind !== "labeled" || !aligned.row) {
+				quarantined++
+
+				continue
+			}
+
+			write(JSON.stringify({ ...aligned.row, synth_method: "cn-organizational-units", synth_base_id: null }) + "\n")
+
+			emitted++
+		}
+
+		console.error(
+			`  cn-organizational-units: ${emitted} labeled, ${skipped} without a chain, ${quarantined} quarantined`
+		)
+
+		return { emitted, skipped, quarantined }
+	},
+}