@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/sg/recipes/register.ts

@@ -3,35 +3,22 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `sg-register` slice recipe — the forms a person in Singapore types, rendered over the Overture-SG register rows
- *   (#2204 §4). The base `overture` source carries the register's own shape, `990 OLD CHOA CHU KANG ROAD, 699814,
- *   Singapore`, and the Latin model has no register for the three forms that dominate typed Singapore addresses:
- *
- *   - the HDB block line, `Blk 123 Ang Mo Kio Ave 3 #05-67 Singapore 560123` — `Blk` in front of the block number (the
- *     register's `number`), a floor-unit `#NN-NN` after the street, the country name as the locality;
- *   - the bracketed postcode, `123 Ang Mo Kio Ave 3 S(560123)` — the form the postal service prints;
- *   - the building-led line, `National Shooting Centre, 990 Old Choa Chu Kang Road, Singapore 699814` — the register's
- *     `unit` field holds the BUILDING or ESTATE name on 91,818 of 142,210 rows, which the adapter refuses as a unit and
- *     this recipe renders as a `venue` when it reads as a building rather than an estate.
- *
- *   Every value in `components` is a verbatim substring of `raw`, aligned by {@link alignAndWrite}. The floor-unit is
- *   SYNTHESIZED (the register carries one on a single row): floor 01–30, unit 01–399, zero-padded — the shape, not a
- *   real occupancy. `Blk` and the `S(` `)` around a postcode are untagged, as the board's `sg-cs-blk-12-kallang-ave`
- *   row expects (`house_number: "12"`). Streets arrive upper-case and are rendered title-cased; a third abbreviate the
- *   generic (`Avenue` → `Ave`, `Road` → `Rd`, …) the way a typed line does.
- *
- *   Run: mailwoman corpus slice sg-register --input <overture-sg.corpus.jsonl> --count N --seed S
+ *   Renders Overture Singapore address rows in the forms that people type, such as HDB block lines and `S(560123)` postcodes.
  */
 
+import { stringifyJSON } from "@mailwoman/core/json"
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
+import { titleCaseIfUpper } from "spliterator"
 
-import { titlecase } from "#au/adapters/gnaf/assemble"
-import { alignAndWrite, type CorpusRecipe, readTuples, sliceSourceID } from "#recipes/scaffold"
+import { alignAndWrite, type CorpusRecipe, readTuples, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { SurfaceOrigin } from "#types"
 
-const SOURCE = "synth-sg-register"
+const SOURCE = defaultRecipeSource("synth-sg-register")
 
 /**
- * The register's generics and the abbreviation a typed line uses for each.
+ * Street generics and their typed abbreviations.
  */
 const GENERIC_ABBREVIATIONS: ReadonlyArray<readonly [full: string, short: string]> = [
 	["Avenue", "Ave"],
@@ -48,27 +35,26 @@ const GENERIC_ABBREVIATIONS: ReadonlyArray<readonly [full: string, short: string
 ]
 
 /**
- * Estate and area names the register's `unit` field carries beside building names. A name ending in one of these is an
- * estate, which no one writes in front of an address line; a name without one reads as a building and is rendered as a
- * `venue`.
+ * Name endings that mark an estate or area in the source `unit` field.
+ *
+ * People do not write an estate name before an address line, so the recipe
+ * renders only building names as a `venue`.
  */
 const ESTATE_TAILS = ["ESTATE", "CONSERVATION AREA", "HILLS", "PARK", "GARDENS", "GARDEN", "HEIGHTS", "GROVE", "VILLE"]
 
-/**
- * Share of rows per register. The block line and the bracketed postcode are the two the Latin model has never seen; the
- * official line keeps the register's own shape in the mix so the block forms train beside it.
- */
+// The official form receives the probability left after the block, bracketed-postcode and building-led forms.
 const P_BLOCK = 0.4
 const P_BRACKET_POSTCODE = 0.25
 const P_BUILDING_LED = 0.15
 const P_ABBREVIATE = 0.35
 const P_UPPER = 0.1
 
+// The source rarely includes a floor-unit, so the recipe synthesizes one within these bounds.
 const FLOOR_MAX = 30
 const UNIT_MAX = 399
 
 /**
- * The draw value that lands each register in the branch order below, for a caller that names the form.
+ * The draw value that selects each form in {@link renderSGRegister} when a caller requests that form.
  */
 const REGISTER_DRAW: Record<SGRegister, number> = {
 	block: 0,
@@ -78,15 +64,8 @@ const REGISTER_DRAW: Record<SGRegister, number> = {
 }
 
 /**
- * `OLD CHOA CHU KANG ROAD` → `Old Choa Chu Kang Road`; an already-mixed value is returned as it is. The register writes
- * every street and building name upper-case, the same convention as G-NAF, so the G-NAF title-caser serves both.
- */
-export function titleCaseSGName(name: string): string {
-	return name === name.toUpperCase() ? titlecase(name) : name
-}
-
-/**
- * Abbreviate the street's generic, wherever it sits (`Ang Mo Kio Avenue 3` → `Ang Mo Kio Ave 3`).
+ * Abbreviates the first street generic that the street contains, such as
+ * `Ang Mo Kio Avenue 3` to `Ang Mo Kio Ave 3`.
  */
 export function abbreviateSGStreet(street: string): string {
 	for (const [full, short] of GENERIC_ABBREVIATIONS) {
@@ -99,8 +78,9 @@ export function abbreviateSGStreet(street: string): string {
 }
 
 /**
- * Whether the register's `unit` value names a building rather than an estate — the building-led register's lead. A name
- * carrying a digit, a parenthesized abbreviation (`DIABETES & METABOLISM CENTRE (DMC)`) or one word only is not one.
+ * Reports whether the source `unit` value looks like a building name that can lead a line.
+ *
+ * The check rejects `NIL`, single words, values with a digit or parenthesis, plus estate names.
  */
 export function isBuildingName(unit: string): boolean {
 	const value = unit.trim().toUpperCase()
@@ -118,12 +98,12 @@ interface SGRow {
 }
 
 /**
- * The four typed forms this recipe renders.
+ * The four typed forms that this recipe renders.
  */
 export type SGRegister = "official" | "block" | "bracket_postcode" | "building_led"
 
 /**
- * One rendered register: the raw line plus the components it carries, every value a substring of the line.
+ * One rendered line with its components, each of which is a substring of `raw`.
  */
 export interface SGRegisterRendering {
 	raw: string
@@ -132,15 +112,19 @@ export interface SGRegisterRendering {
 }
 
 /**
- * Render one register form for one register row. Pure: every random draw comes through `random`. A `register` names the
- * form to render instead of drawing one, for a board that wants a balanced spread; `building_led` still needs a
- * building-shaped `unit` and falls back to `official` without one.
+ * Renders one source row in a sampled or requested form.
+ *
+ * All randomness comes from `random`.
+ * A requested `building_led` form falls back to `official` when the row's `unit` is not a building name.
+ *
+ * A requested `block` form falls back to `bracket_postcode` when the row's number is not a block number.
+ * `Blk` and the `S(` and `)` around a postcode stay untagged.
  */
 export function renderSGRegister(row: SGRow, random: () => number, register?: SGRegister): SGRegisterRendering {
-	const street0 = titleCaseSGName(row.street)
+	const street0 = titleCaseIfUpper(row.street)
 	const street = random() < P_ABBREVIATE ? abbreviateSGStreet(street0) : street0
 	const drawn = random()
-	const building = isBuildingName(row.unit) ? titleCaseSGName(row.unit) : null
+	const building = isBuildingName(row.unit) ? titleCaseIfUpper(row.unit) : null
 	const draw = register === undefined ? drawn : REGISTER_DRAW[register]
 
 	let raw: string
@@ -182,7 +166,9 @@ export function renderSGRegister(row: SGRow, random: () => number, register?: SG
 }
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the registers and why each is here.
+ * Renders each Overture Singapore corpus row from `--input` once in a sampled typed form.
+ *
+ * The recipe skips rows without a street, a block-style number, or a six-digit postcode.
  */
 export const sgRegisterRecipe: CorpusRecipe = {
 	name: "sg-register",
@@ -227,7 +213,7 @@ export const sgRegisterRecipe: CorpusRecipe = {
 			if (opts.golden) {
 				const golden = { raw: rendering.raw, components: rendering.components, country: "SG", locale: "en-SG" }
 
-				write(JSON.stringify(golden) + "\n")
+				write(stringifyJSON(golden))
 
 				emitted++
 				registers[rendering.register] = (registers[rendering.register] ?? 0) + 1
@@ -243,12 +229,20 @@ export const sgRegisterRecipe: CorpusRecipe = {
 					country: "SG",
 					locale: "en-SG",
 					source,
-					source_id: sliceSourceID(source, { raw: rendering.raw }),
+					source_id: recipeSourceID(source, { raw: rendering.raw }),
 					corpus_version: "0.1.0",
 					license:
 						"CDLA-Permissive-2.0 — Overture Maps addresses over the Singapore Open Data Licence 1.0 (OneMap / Singapore Land Authority)",
 				},
-				`sg-register:${rendering.register}`
+				`sg-register:${rendering.register}`,
+				// The values come from Overture and the recipe chose only their arrangement,
+				// so the Overture record behind each row is what `baseSourceID` names
+				// when the tuples file includes its id.
+				{
+					register: SourceRegister.Overture,
+					surface: SurfaceOrigin.Composed,
+					baseSourceID: tuple.sourceID ?? null,
+				}
 			)
 
 			if (ok) {

package/lib/sg/tools/fetch/acra.ts

@@ -3,19 +3,8 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Fetch Singapore's ACRA corporate-entity register from data.gov.sg: 27 monthly CSVs (one per first
- *   letter of the entity name plus "others"), 53 columns, the registered address FIELDED — `Block`,
- *   `Street Name`, `Level No`, `Unit No`, `Building Name`, `Postal Code` — which is the NPPES shape,
- *   so the corpus adapter renders the string and the spans fall out by construction. A Singapore
- *   postcode names one building, so the postcode column is also the join to the OneMap/Overture rows.
- *
- *   License: Singapore Open Data Licence version 1.0 (attribution; commercial use, modification and
- *   adaptation permitted; no share-alike). The prescribed attribution sentence goes in the manifest.
- *
- *   data.gov.sg serves a dataset in two calls: `initiate-download` prepares a signed URL and
- *   `poll-download` answers it once the export is ready.
- *
- *   Invoke via `mailwoman corpus fetch acra-sg --out-root <path>`.
+ *   Fetches Singapore ACRA corporate-entity CSVs from data.gov.sg.
+ *   The publisher includes fielded addresses in these files.
  */
 
 import { BYTES_PER_KIB } from "@mailwoman/core/fs/formatters"
@@ -23,15 +12,9 @@ import { pathExists } from "@mailwoman/core/fs/readers"
 import { makeDirectories } from "@mailwoman/core/fs/writers"
 import { sha256File } from "@mailwoman/core/hash"
 import { sleep } from "@mailwoman/core/utils/sleep"
-import { join } from "path-ts"
 
-import type {
-	BaseFetchOptions,
-	FetchSummary,
-	SourceCollectionManifest,
-	SourceManifest,
-} from "#tools/fetch/download/index"
-import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download/index"
+import type { BaseFetchOptions, FetchSummary, SourceCollectionManifest, SourceManifest } from "#tools/fetch/download"
+import { loadCollectionFiles, streamBodyToFile, withRetries, writeManifest } from "#tools/fetch/download"
 
 const SLUG = "acra-sg"
 const COLLECTION_URL = "https://api-production.data.gov.sg/v2/public/api/collections/2/metadata"
@@ -39,12 +22,17 @@ const DATASET_API = "https://api-production.data.gov.sg/v2/public/api/datasets"
 const DOWNLOAD_API = "https://api-open.data.gov.sg/v1/public/api/datasets"
 const LICENSE = "Singapore Open Data Licence version 1.0 — https://data.gov.sg/open-data-licence"
 
+// The license prescribes this attribution sentence.
+// The manifest records it.
 const ATTRIBUTION =
 	"Contains information from ACRA Information on Corporate Entities accessed on <date> from data.gov.sg which is made available under the terms of the Singapore Open Data Licence version 1.0 https://data.gov.sg/open-data-licence"
 
 const POLL_INTERVAL_MS = 3000
 const POLL_ATTEMPTS = 40
 
+/**
+ * Options for {@link fetchACRASG}.
+ */
 export type FetchACRASGOptions = BaseFetchOptions
 
 interface CollectionMetadata {
@@ -68,7 +56,9 @@ async function readJSON<T>(url: string): Promise<T> {
 }
 
 /**
- * Ask the portal to prepare the CSV export, then poll until it names the signed URL.
+ * Asks data.gov.sg to prepare a CSV export and polls until the export's signed URL is ready.
+ *
+ * The function returns `undefined` when polling runs out of attempts.
  */
 async function signedURLFor(datasetID: string): Promise<string | undefined> {
 	await readJSON(`${DOWNLOAD_API}/${datasetID}/initiate-download`)
@@ -89,10 +79,15 @@ function filenameFor(name: string, datasetID: string): string {
 	return `acra-entities-${(letter ?? datasetID).toLowerCase().replaceAll(/[^a-z0-9]+/g, "-")}.csv`
 }
 
+/**
+ * Downloads every dataset in the ACRA collection and writes a collection manifest after each file.
+ *
+ * The function skips a file that the previous manifest lists and that still exists on disk.
+ */
 export async function fetchACRASG(options: FetchACRASGOptions, report?: (line: string) => void): Promise<FetchSummary> {
-	const destDir = join(options.outRoot, SLUG)
+	const destDir = options.outRoot(SLUG)
 	await makeDirectories(destDir)
-	const manifestPath = join(destDir, "MANIFEST.json")
+	const manifestPath = destDir("MANIFEST.json")
 
 	const collection = await readJSON<CollectionMetadata>(COLLECTION_URL)
 	const datasetIDs = collection.data?.collectionMetadata?.childDatasets ?? []
@@ -108,7 +103,7 @@ export async function fetchACRASG(options: FetchACRASGOptions, report?: (line: s
 		const metadata = await readJSON<DatasetMetadata>(`${DATASET_API}/${datasetID}/metadata`)
 		const name = metadata.data?.name ?? datasetID
 		const filename = filenameFor(name, datasetID)
-		const dest = join(destDir, filename)
+		const dest = destDir(filename)
 		const before = previous.get(filename)
 
 		if (before && (await pathExists(dest))) {

package/lib/si/recipes/bare-village.ts

@@ -3,34 +3,35 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `si-bare-village` — the Slovenian no-street counter-slice (#901 run-2). Slovenia's rural
- *   addressing has NO street line: the village name is the street-level token and repeats as the
- *   locality ("Zabiče 8, 6250 Zabiče"). The v1.9.8 check FALSIFIED the fr-bare-street slice alone on
- *   exactly this class (SI resolve −3.4pp; "Apače 108" split into street "Apače 10" + house "8") —
- *   the bare-street boundary lesson generalizes onto a form where the leading name must keep its
- *   number whole and the trailing mention must stay locality-bound. This slice is the paired
- *   counter-distribution: same lesson ("name before number, comma, then admin"), opposite polarity
- *   on the trailing mention.
+ *   `si-bare-village`, the Slovenian no-street counter-recipe. Slovenia's rural addressing has no
+ *   street line, so the village name is the street-level token and repeats as the locality
+ *   ("Zabiče 8, 6250 Zabiče"). The leading name must keep its number whole. The trailing mention
+ *   must stay locality-bound. This recipe is the paired counter-distribution, with the same lesson
+ *   ("name before number, comma, then admin") and the opposite polarity on the trailing mention.
  *
  *   Three real-order templates cycle per tuple (mirrors the coord-golden orders so the eval and the
  *   training distribution agree):
  *
  *   1. canonical  "«V» «n», «pc» «V»"
- *   2. bare       "«V» «n», «V»"        (no postcode — the anchor-free form, the fr-bare lesson)
+ *   2. bare       "«V» «n», «V»"        (no postcode, the anchor-free form)
  *   3. pc-first   "«pc» «V», «V» «n»"
  *
- *   Gold spans: leading «V» = street (matches OA ground truth — the village IS the address line),
- *   «n» = house_number (NEVER split mid-digits), «pc» = postcode (never swallowing the neighbor),
+ *   Gold spans: leading «V» = street (matches OA ground truth, the village is the address line),
+ *   «n» = house_number (never split mid-digits), «pc» = postcode (never swallowing the neighbor),
  *   trailing «V» = locality (the binding the resolver needs).
  */
 
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 
-import { alignAndWrite, readTuples, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import { alignAndWrite, readTuples, requireRegister, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SurfaceOrigin } from "#types"
 
 /**
- * Slice recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
- * and `description` below for the surface form it generates.
+ * Recipe registered with the corpus builder.
+ *
+ * See the file header for the parse behavior this recipe exercises
+ * and `description` below for its surface form.
  */
 export const siBareVillageRecipe: CorpusRecipe = {
 	name: "si-bare-village",
@@ -42,6 +43,11 @@ export const siBareVillageRecipe: CorpusRecipe = {
 		let emitted = 0
 		let skipped = 0
 
+		const SI_BARE_VILLAGE_PROVENANCE = {
+			register: requireRegister(opts, "si-bare-village"),
+			surface: SurfaceOrigin.Composed,
+		}
+
 		for await (const t of readTuples(opts.input!)) {
 			read++
 			const village = String(t.locality ?? "").trim()
@@ -73,21 +79,32 @@ export const siBareVillageRecipe: CorpusRecipe = {
 				raw = `${postcode} ${village}, ${village} ${number}`
 			}
 
-			const source_id = sliceSourceID("synth-si-bare-village", { ...components, o: String(order), v: String(read) })
+			// One resolution for both, because `source_id` includes the source as its prefix
+			// and a pair that disagreed would name a source no row of this output declares.
+			const source = defaultRecipeSource("synth-si-bare-village")
+			const source_id = recipeSourceID(source, { ...components, o: String(order), v: String(read) })
 
 			const canonical = {
 				raw,
 				components,
 				country: "SI",
 				locale: "sl-SI",
-				source: "synth-si-bare-village",
+				source,
 				source_id,
 				corpus_version: "0.9.9",
 				license:
 					"Synthetic — si-bare-village; (village, number, postcode) from OpenAddresses SI (per-source attribution in the model card)",
 			}
 
-			if (alignAndWrite(write, canonical, "si-bare-village")) {
+			// Per row rather than on the shared literal, because the id names this tuple's record.
+			// A tuples file written before `sourceID` existed has no id.
+			// `null` records that case.
+			if (
+				alignAndWrite(write, canonical, "si-bare-village", {
+					...SI_BARE_VILLAGE_PROVENANCE,
+					baseSourceID: t.sourceID ?? null,
+				})
+			) {
 				emitted++
 			} else {
 				skipped++