@mailwoman/corpus 9.2.0 → 9.3.0

package/lib/recipes/po-box.ts

@@ -0,0 +1,123 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `po-box` slice recipe — synthetic PO box rows: tuples → {@link synthesizePoBoxRow} → aligned
+ *   LabeledRow, plus optional self-contained US military/diplomatic rows (#517) at
+ *   `--military-ratio`. Region is required EXCEPT region-less locales (NZ). Ported from
+ *   scripts/build-po-box-slice.mjs.
+ */
+
+import {
+	alignAndWrite,
+	makeLcg,
+	readTuples,
+	sliceSourceID,
+	SYNTHETIC_TUPLE_LICENSE as LICENSE,
+	type CorpusRecipe,
+} from "#recipes/scaffold"
+import { synthesizeMilitaryPoBoxRow, synthesizePoBoxRow, type PoBoxBaseTuple } from "#synthesizers/po-box"
+
+/**
+ * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
+ * and `description` below for the surface form it generates.
+ */
+export const poBoxRecipe: CorpusRecipe = {
+	name: "po-box",
+	description: "PO box rows: tuples → synthesizePoBoxRow (+ optional US military/diplomatic rows)",
+	mode: "tuples",
+	options: [
+		{ flag: "--pmb-ratio <p>", description: "P(private-mailbox layout). Default 0.15" },
+		{ flag: "--military-ratio <p>", description: "P(emit one US military/diplomatic row per input, #517). Default 0" },
+	],
+	async run(opts, write) {
+		if (!opts.input) throw new Error("po-box recipe requires --input <tuples.jsonl>")
+		const random = makeLcg(opts.seed)
+		const pmbRatio = opts.pmbRatio ?? 0.15
+		const militaryRatio = opts.militaryRatio ?? 0
+		let read = 0
+		let emitted = 0
+		let skipped = 0
+
+		for await (const tuple of readTuples(opts.input)) {
+			read++
+			// Region required EXCEPT region-less locales (NZ: "Private Bag 12, Auckland 1010", #517).
+			const regionOptional = ["NZ", "NZL", "NEW ZEALAND"].includes(String(tuple.country || "").toUpperCase())
+
+			if (!tuple.locality || !tuple.postcode || !tuple.country || (!tuple.region && !regionOptional)) {
+				skipped++
+
+				continue
+			}
+
+			for (let v = 0; v < opts.variants; v++) {
+				const synth = synthesizePoBoxRow(tuple as PoBoxBaseTuple, { random, pmbRatio })
+
+				if (!synth) continue
+
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: synth.raw,
+						components: synth.components,
+						country: tuple.country,
+						locale: synth.locale,
+						source: "synth-po-box",
+						source_id: sliceSourceID("synth-po-box", {
+							locality: tuple.locality,
+							region: tuple.region,
+							postcode: tuple.postcode,
+							country: tuple.country,
+							v: String(v),
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					synth.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+
+			// US military/diplomatic rows (#517): self-contained, one per input line at --military-ratio.
+			// Default 0 → byte-stable (random() not called when off). US-only.
+			if (militaryRatio > 0 && random() < militaryRatio) {
+				const mil = synthesizeMilitaryPoBoxRow({ random })
+
+				const ok = alignAndWrite(
+					write,
+					{
+						raw: mil.raw,
+						components: mil.components,
+						country: "US",
+						locale: mil.locale,
+						source: "synth-po-box",
+						source_id: sliceSourceID("synth-po-box", {
+							po_box: mil.components.po_box,
+							locality: mil.components.locality,
+							region: mil.components.region,
+							postcode: mil.components.postcode,
+							v: `mil${emitted}`,
+						}),
+						corpus_version: "0.4.0",
+						license: LICENSE,
+					},
+					mil.template
+				)
+
+				if (ok) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read, emitted, skipped }
+	},
+}

package/lib/recipes/reviewed-postcode-tail.ts

@@ -0,0 +1,190 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   A bounded, reviewed source for Venezuela's `locality postcode, region` convention (#1821).
+ *   Bulk sources yielded zero Venezuelan tuples, so this recipe reads four reviewed facts and applies
+ *   only transformations that do not create another postcode-to-place join.
+ */
+
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { resolveModulePath } from "@mailwoman/core/module/resolvers"
+import { stripCombiningMarks } from "@mailwoman/normalize/fold"
+
+import { alignAndWrite, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+
+/**
+ * A separate sampler bucket so a receipt measures these reviewed after-locality rows and no other postcode placement.
+ */
+export const REVIEWED_POSTCODE_TAIL_SOURCE = "synth-reviewed-postcode-tail"
+
+/**
+ * The four postcode-to-place facts reviewed for #1821 and committed in the package data file.
+ */
+const REVIEWED_TUPLE_COUNT = 4
+
+interface ReviewedTupleProvenance {
+	publisher: string
+	url: string
+	retrievedAt: string
+	reviewStatus: "reviewed"
+	sourceLicenseNote: string
+}
+
+export interface ReviewedPostcodeTuple {
+	id: string
+	locality: string
+	postcode: string
+	region: string
+	country: "Venezuela"
+	cc: "VE"
+	locale: "es-VE"
+	postcodePlacement: "after_locality"
+	provenance: ReviewedTupleProvenance
+}
+
+interface ReviewedPostcodeTupleFile {
+	version: string
+	reviewedAt: string
+	tuples: ReviewedPostcodeTuple[]
+}
+
+type VariantID =
+	| "canonical-country"
+	| "canonical-no-country"
+	| "comma-free"
+	| "uppercase"
+	| "left-context"
+	| "accent-folded"
+
+interface Variant {
+	id: VariantID
+	raw: string
+	components: Record<string, string>
+}
+
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
+
+export function defaultReviewedPostcodeTuplePath(): string {
+	return resolveModulePath("@mailwoman/corpus/data/reviewed-ve-postcode-tuples.json")
+}
+
+export async function readReviewedPostcodeTuples(
+	path = defaultReviewedPostcodeTuplePath()
+): Promise<ReviewedPostcodeTuple[]> {
+	const document = await readLocalJSONFile<ReviewedPostcodeTupleFile>(path)
+
+	if (!document.version || !ISO_DATE.test(document.reviewedAt) || document.tuples.length !== REVIEWED_TUPLE_COUNT) {
+		throw new Error("reviewed postcode tuple file must declare a version, review date, and exactly four tuples")
+	}
+
+	const ids = new Set<string>()
+	const facts = new Set<string>()
+
+	for (const tuple of document.tuples) {
+		const provenance = tuple.provenance
+		const fact = `${tuple.locality}\u0000${tuple.postcode}\u0000${tuple.region}`
+
+		if (
+			!tuple.id ||
+			!tuple.locality ||
+			!tuple.postcode ||
+			!tuple.region ||
+			tuple.country !== "Venezuela" ||
+			tuple.cc !== "VE" ||
+			tuple.locale !== "es-VE" ||
+			tuple.postcodePlacement !== "after_locality" ||
+			!provenance?.publisher ||
+			!URL.canParse(provenance.url) ||
+			!ISO_DATE.test(provenance.retrievedAt) ||
+			provenance.reviewStatus !== "reviewed" ||
+			!provenance.sourceLicenseNote
+		) {
+			throw new Error(`invalid reviewed postcode tuple: ${tuple.id || "<missing id>"}`)
+		}
+
+		if (ids.has(tuple.id) || facts.has(fact)) throw new Error(`duplicate reviewed postcode tuple: ${tuple.id}`)
+		ids.add(tuple.id)
+		facts.add(fact)
+	}
+
+	return document.tuples
+}
+
+const foldAccents = (value: string): string => stripCombiningMarks(value).normalize("NFC")
+
+export function reviewedPostcodeTailVariants(tuple: ReviewedPostcodeTuple): Variant[] {
+	const tail = `${tuple.locality} ${tuple.postcode}, ${tuple.region}`
+	const withCountry = `${tail}, ${tuple.country}`
+	const base = { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region, country: tuple.country }
+	const upper = Object.fromEntries(Object.entries(base).map(([key, value]) => [key, value.toUpperCase()]))
+
+	const variants: Variant[] = [
+		{ id: "canonical-country", raw: withCountry, components: base },
+		{
+			id: "canonical-no-country",
+			raw: tail,
+			components: { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region },
+		},
+		{ id: "comma-free", raw: `${tuple.locality} ${tuple.postcode} ${tuple.region} ${tuple.country}`, components: base },
+		{ id: "uppercase", raw: withCountry.toUpperCase(), components: upper },
+		{
+			id: "left-context",
+			raw: `Comercio Ejemplo, Calle Principal, ${withCountry}`,
+			components: { venue: "Comercio Ejemplo", street: "Calle Principal", ...base },
+		},
+	]
+
+	const foldedLocality = foldAccents(tuple.locality)
+	const foldedRegion = foldAccents(tuple.region)
+
+	if (foldedLocality !== tuple.locality || foldedRegion !== tuple.region) {
+		variants.push({
+			id: "accent-folded",
+			raw: `${foldedLocality} ${tuple.postcode}, ${foldedRegion}, ${tuple.country}`,
+			components: { locality: foldedLocality, postcode: tuple.postcode, region: foldedRegion, country: tuple.country },
+		})
+	}
+
+	return variants
+}
+
+/**
+ * Emit bounded surface variants of the committed reviewed facts without constructing another geographic join.
+ */
+export const reviewedPostcodeTailRecipe: CorpusRecipe = {
+	name: "reviewed-postcode-tail",
+	description: "Reviewed Venezuela locality-postcode-region tails with bounded surface variants",
+	mode: "generate",
+	async run(opts, write) {
+		const tuples = await readReviewedPostcodeTuples(opts.input)
+		let emitted = 0
+		let skipped = 0
+
+		for (const tuple of tuples) {
+			for (const variant of reviewedPostcodeTailVariants(tuple)) {
+				const sourceID = sliceSourceID(REVIEWED_POSTCODE_TAIL_SOURCE, { tuple: tuple.id, variant: variant.id })
+
+				const canonical = {
+					raw: variant.raw,
+					components: variant.components,
+					country: tuple.cc,
+					locale: tuple.locale,
+					source: REVIEWED_POSTCODE_TAIL_SOURCE,
+					source_id: sourceID,
+					corpus_version: "0.11.0",
+					license: `Synthetic rendering of reviewed postal facts — ${tuple.provenance.publisher}; source terms recorded in reviewed-ve-postcode-tuples.json`,
+				}
+
+				if (alignAndWrite(write, canonical, "reviewed-postcode-tail", tuple.id)) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read: tuples.length, emitted, skipped }
+	},
+}