@mailwoman/corpus 10.1.0 → 10.2.1

package/lib/recipes/scaffold.ts

@@ -9,7 +9,7 @@
  *   supplies the I/O.
  */
 
-import { pathExists } from "@mailwoman/core/fs/readers"
+import { pathExists } from "@mailwoman/core/fs/readers/stat"
 import { readZipEntry } from "@mailwoman/core/fs/zip"
 import { tryParsingJSON, stringifyJSON } from "@mailwoman/core/json"
 import type { PathBuilderLike } from "path-ts"
@@ -59,9 +59,10 @@ export interface RecipeTuple {
 	postcode?: string
 	country?: string
 	/**
-	 * Defaults to `leading` when absent.
 	 *
 	 * That matches every tuples file written before this field existed.
+	 *
+	 * @defaultValue `leading` when absent.
 	 */
 	postcodePlacement?: PostcodePlacement
 	/**
@@ -351,7 +352,8 @@ export function alignAndWrite(
 ): boolean {
 	const aligned = alignRow(canonical as Parameters<typeof alignRow>[0])
 
-	if (!aligned.row) return false
+	// A quarantined result also has a `row`, the quarantine record, so the kind decides.
+	if (aligned.kind !== "labeled") return false
 
 	write(
 		stringifyJSON({
@@ -395,26 +397,28 @@ export interface RecipeOptions {
 	intlFraction?: number
 	/**
 	 * `german`: fraction of native-order rows rendered with no commas at all.
-	 * The default is 0.3.
+	 *
+	 * @defaultValue `0.3`
 	 */
 	commaFreeFraction?: number
 	/**
 	 * `german`: fraction of rows whose `dependent_locality` is a WOF Ortsteil of the tuple's locality.
 	 *
-	 * The default is 0.3.
 	 * It is 0 when no admin database is readable.
+	 *
+	 * @defaultValue `0.3`
 	 */
 	ortsteilFraction?: number
 	/**
 	 * `german`: the WOF admin database the Ortsteil pool is read from.
 	 *
-	 * Default `$MAILWOMAN_DATA_ROOT/db/wof/admin-global-priority-importance.db`.
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/db/wof/admin-global-priority-importance.db`
 	 */
 	adminDB?: string
 	/**
 	 * `locale`: fraction of rows that append an explicit country surface form + a `country` component.
 	 *
-	 * Default 0.
+	 * @defaultValue `0`
 	 */
 	countryFraction?: number
 	/**
@@ -430,7 +434,7 @@ export interface RecipeOptions {
 	/**
 	 * `fr-lieudit`: BAN `adresses-<dept>.csv` directory.
 	 *
-	 * Default `$MAILWOMAN_DATA_ROOT/corpus/sources/ban`.
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/corpus/sources/ban`
 	 */
 	banDir?: PathBuilderLike
 	multilocaleCount?: number
@@ -449,48 +453,54 @@ export interface RecipeOptions {
 	 * `no-fragment` knob 3: emit N copies of each street+number row whose number has >=
 	 * longNumberMinDigits digits (oversample the failing long-number class).
 	 *
-	 * Default 1 = no boost.
+	 * @defaultValue `1` = no boost
 	 */
 	longNumberBoost?: number
 	/**
 	 * `no-fragment` knob 3: minimum digit count for a number to count as "long" and be boosted.
 	 *
-	 * Default 3.
+	 * @defaultValue `3`
 	 */
 	longNumberMinDigits?: number
 	/**
 	 * `sub-venue`: the sub-venue lexicon JSON.
 	 *
-	 * Default = the committed `corpus/data/sub-venue-lexicon.json`, resolved through the package manifest so it works
+	 * @defaultValue = the committed `corpus/data/sub-venue-lexicon.json`, resolved through the package manifest so it works
 	 * from the source tree and from `out/`.
 	 */
 	lexicon?: string
 	/**
 	 * `sub-venue`: directory of `sub-venue-extract` JSONLs, one per region.
 	 *
-	 * Default `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`.
+	 * @defaultValue `$MAILWOMAN_DATA_ROOT/sub-venue/extracts`
 	 */
 	extractsDir?: string
-	/**
-	 * `sub-venue`: the `poi.db` spatial layer, read for the en-US and fr-FR venue + confound
-	 * pools (the two of poi.db's four countries this recipe has legs for).
-	 *
-	 * Default `$MAILWOMAN_DATA_ROOT/db/poi/poi.db`.
-	 */
-	poiDB?: string
 	/**
 	 * `sub-venue`: GB/US/FR address-context tuples jsonl.
 	 *
-	 * Default the house-venue v3 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`);
+	 * @defaultValue the house-venue v3 tuples (`$MAILWOMAN_DATA_ROOT/corpus/intermediate/house-venue-tuples-v3.jsonl`);
 	 * DE and ES read OpenAddresses directly.
 	 */
 	subVenueTuples?: string
 	/**
 	 * `sub-venue`: share of emitted rows that are confound negatives.
 	 *
-	 * Default 0.3.
+	 * @defaultValue `0.3`
 	 */
 	negativeFraction?: number
+	/**
+	 * `venue` and `sub-venue`: the Overture place-names Parquet file with `name`,
+	 * `category` and `country` columns.
+	 *
+	 * @defaultValue the release the venue-head table is built from, under `$MAILWOMAN_DATA_ROOT/overture`
+	 */
+	venueNames?: PathBuilderLike
+	/**
+	 * `venue`: share of locality rows that also carry the tuple's region after the locality.
+	 *
+	 * @defaultValue `0.5`
+	 */
+	venueRegionFraction?: number
 }
 
 /**

package/lib/recipes/sources.ts

@@ -91,7 +91,7 @@ export interface RecipeSource {
 	/**
 	 * Why the producer's path does not fully explain the placement.
 	 */
-	note?: string
+	note: string | null
 }
 
 /**
@@ -107,7 +107,7 @@ function entries(
 		current: `${prefix}-${tail}`,
 		operation,
 		producer,
-		...(note ? { note } : {}),
+		note: note || null,
 	}))
 }
 
@@ -164,6 +164,7 @@ export const RECIPE_SOURCES: ReadonlyArray<RecipeSource> = [
 		current: "rendered-de",
 		operation: SourceOperation.Rendered,
 		producer: "de/recipes/locale.ts",
+		note: null,
 	},
 	// Three tails include the word `fragment` themselves.
 	// It folds into the prefix instead of repeating.
@@ -183,21 +184,33 @@ export const RECIPE_SOURCES: ReadonlyArray<RecipeSource> = [
 		current: "fragment-fr",
 		operation: SourceOperation.Fragment,
 		producer: "fr/recipes/fragment.ts",
+		note: null,
 	},
 	{
 		retired: "synth-no-fragment",
 		current: "fragment-no",
 		operation: SourceOperation.Fragment,
 		producer: "no/recipes/fragment.ts",
+		note: null,
 	},
 	...entries(SourceOperation.Fragment, "fragment", [
 		["fr-bare-street", "fr/recipes/bare-street.ts"],
 		["fr-bare-street-v20", "fr/recipes/bare-street.ts"],
 		["fr-bare-street-v21", "fr/recipes/bare-street.ts"],
 		["fr-bare-street-v22", "fr/recipes/bare-street.ts"],
+		[
+			"venue-bare",
+			"recipes/venue.ts",
+			"An Overture place name as the whole query. The retired spelling never reached a corpus: the recipe was written after the rename.",
+		],
 	]),
 	...entries(SourceOperation.Spliced, "spliced", [
 		["unit", "recipes/unit.ts", "The unit designator and number are drawn."],
+		[
+			"venue-locality",
+			"recipes/venue.ts",
+			"The locality is drawn from the country's admin tuples. The retired spelling never reached a corpus.",
+		],
 		["unit-v30", "recipes/unit.ts"],
 		["affix", "recipes/street/affix.ts"],
 		["suffix-boundary", "recipes/street/affix.ts"],
@@ -259,7 +272,7 @@ export const CARRIED_SOURCES: ReadonlyArray<string> = [
 	"gnaf",
 	"osm",
 	// One id per country for the 2026-10-02 Geofabrik extracts, written by `corpus run osm --source-name`.
-	// A country's OSM rows then carry their own weight, so admitting a country,
+	// A country's OSM rows then hold their own weight, so admitting a country,
 	// admitting its OSM rows and sampling them stay three settings.
 	// The September PK, VN and BD rows keep `osm`.
 	"osm-ru",

package/lib/recipes/street/bare.ts

@@ -22,9 +22,9 @@ import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 
 import { recipeSourceID, requireRegister, type CanonicalRecipeRow, type CorpusRecipe } from "#recipes/scaffold"
 import { defaultRecipeSource } from "#recipes/sources"
-import { DEFAULT_US_BASES } from "#synthesizers/intersection"
-import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
 import { SurfaceOrigin } from "#types"
+import { DEFAULT_US_BASES } from "#us/synthesizers/intersection"
+import { synthesizeStreetRow, type StreetBaseTuple } from "#us/synthesizers/street"
 import { alignRow } from "#utils"
 
 /**

package/lib/recipes/street.ts

@@ -12,8 +12,8 @@ import { makeLcg } from "@mailwoman/core/utils"
 import { stableSourceID } from "#adapters/source-id"
 import { alignAndWrite, readTuples, type CorpusRecipe } from "#recipes/scaffold"
 import { defaultRecipeSource } from "#recipes/sources"
-import { synthesizeStreetRow, type StreetBaseTuple } from "#synthesizers/street"
 import { SurfaceOrigin } from "#types"
+import { synthesizeStreetRow, type StreetBaseTuple } from "#us/synthesizers/street"
 
 /**
  * The recipe draws a street name from a pool and joins it to a tuple's locality, region and postcode.
@@ -62,6 +62,8 @@ export const streetRecipe: CorpusRecipe = {
 				continue
 			}
 
+			// country-branch: `synthesizeStreetRow` renders the US layout only.
+			// The codex layouts in the `locale` recipe cover the rest.
 			if (tuple.country !== "US") {
 				skipped++
 

package/lib/recipes/sub/venue/context.ts

@@ -138,6 +138,8 @@ export function buildStreetNegatives(
 				isPair = true
 			}
 
+			// country-branch: `-gate` is a street-name suffix in British English only.
+			// The street-morphology FST per locale would replace it.
 			if (country === "GB" && token.length >= MIN_GATE_SUFFIX_TOKEN_LENGTH && token.endsWith("gate")) {
 				gateSuffix.push(tuple)
 			}

package/lib/recipes/sub/venue/render.ts

@@ -150,7 +150,7 @@ export function addressGroups(country: string, tuple: LocaleBaseTuple, withStree
 	let current: Group = []
 
 	for (const piece of renderAddress(layout, components).pieces) {
-		if (piece.tag === null) {
+		if (!piece.tag) {
 			if (GROUP_BREAK.has(piece.text) && current.length) {
 				groups.push(current)
 				current = []

package/lib/recipes/sub/venue/sources.ts

@@ -9,10 +9,9 @@ import { resolveModulePath } from "@mailwoman/core/module/resolvers"
 import { sample } from "@mailwoman/core/random"
 import { escapeRegExp } from "@mailwoman/core/strings/regexp"
 import { upperFirst } from "@mailwoman/normalize/case"
-import type { POIDatabase } from "@mailwoman/resolver-wof-sqlite/poi"
-import { DatabaseClient } from "@mailwoman/sqlite/client"
 import type { PathBuilderLike } from "path-ts"
 
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb"
 import { classifyIdentifier, readSubVenueJSONL } from "#subvenue/harvest"
 import { SUBVENUE_PROMOTIONS, type SubVenuePromotion } from "#subvenue/promotions"
 import type { SubVenueLexiconTable } from "#subvenue/table"
@@ -30,7 +29,7 @@ export function defaultLexiconPath(): string {
  * Reads the sub-venue lexicon and throws on invalid JSON.
  */
 export async function readSubVenueLexicon(path: string = defaultLexiconPath()): Promise<SubVenueLexiconTable> {
-	return await readLocalJSONFile<SubVenueLexiconTable>(path)
+	return readLocalJSONFile<SubVenueLexiconTable>(path)
 }
 
 const MAX_VENUE_NAME_LENGTH = 44
@@ -516,43 +515,39 @@ const POI_CONFOUND_CATEGORIES: readonly string[] = [
 ]
 
 /**
- * Reads the venue and confound name pools for one country from `poi.db`.
+ * Reads the venue and confound name pools for one country from the Overture place-names file,
+ * which holds `name`, `category` and `country` for every country the release covers.
  *
- * The database covers only a few countries, so an empty result may reflect missing coverage.
+ * The venue categories fill the `venues` pool.
+ * Every row of a venue or confound category is tested against the rejected phrases
+ * and the designator phrases, as the OSM extracts are.
  */
-export function readPOIPools(dbPath: PathBuilderLike, country: string, query: PoolQuery): NamePools {
-	using db = new DatabaseClient<POIDatabase>(dbPath, { readOnly: true })
-
-	const codes = db.prepare("select id, category from poi_category_codes").all() as Array<{
-		id: number
-		category: string
-	}>
-
-	const byName = new Map(codes.map((c) => [c.category, c.id]))
-	const venueIDs = POI_VENUE_CATEGORIES.map((c) => byName.get(c)).filter((id): id is number => id != null)
-	const confoundIDs = POI_CONFOUND_CATEGORIES.map((c) => byName.get(c)).filter((id): id is number => id != null)
-	const wanted = [...venueIDs, ...confoundIDs]
-
-	if (!wanted.length) throw new Error(`poi.db at ${dbPath} has none of the expected categories`)
-
-	const rows = db
-		.prepare(
-			`select name, category_id from poi where country = ? and name is not null and category_id in (${wanted.map(() => "?").join(",")})`
-		)
-		.all(country, ...wanted) as Array<{ name: string; category_id: number }>
+export async function readOvertureNamePools(
+	venueNames: PathBuilderLike,
+	country: string,
+	query: PoolQuery
+): Promise<NamePools> {
+	const categories = [...POI_VENUE_CATEGORIES, ...POI_CONFOUND_CATEGORIES].map((c) => `'${escapeSQLString(c)}'`)
+
+	using db = await openDuckDB({ threads: 4 })
+
+	const result = await db.runAndReadAll(
+		`SELECT DISTINCT name, category FROM read_parquet('${escapeSQLString(String(venueNames))}')
+		WHERE country = '${escapeSQLString(country)}' AND name IS NOT NULL AND category IN (${categories.join(",")})`
+	)
 
-	const venueSet = new Set(venueIDs)
+	const venueSet = new Set(POI_VENUE_CATEGORIES)
 	const venues = new Set<string>()
 	const rejectedVenues = new Set<string>()
 	const longerNames = new Set<string>()
 
-	for (const row of rows) {
-		const name = row.name.trim()
+	for (const [nameValue, categoryValue] of result.getRows()) {
+		const name = String(nameValue).trim()
 
 		if (!isVenueSlotName(name)) continue
 		const low = name.toLowerCase()
 
-		if (venueSet.has(row.category_id)) {
+		if (venueSet.has(String(categoryValue))) {
 			venues.add(name)
 		}
 

package/lib/recipes/sub/venue.ts

@@ -12,7 +12,6 @@ import { stringifyJSON } from "@mailwoman/core/json"
 import { sample } from "@mailwoman/core/random"
 import { mulberry32 as makeMulberry32 } from "@mailwoman/core/utils"
 import { upperFirst } from "@mailwoman/normalize/case"
-import { poiDatabasePath } from "@mailwoman/resolver-wof-sqlite/paths"
 import type { PathBuilderLike } from "path-ts"
 
 import { recipeSourceID, type CorpusRecipe } from "#recipes/scaffold"
@@ -30,19 +29,16 @@ import {
 	type PromotedSurface,
 	promotedSurfacesFor,
 	readExtractPools,
-	readPOIPools,
+	readOvertureNamePools,
 	readSubVenueLexicon,
 	rejectedPhrasesFor,
 	sampleIdentifier,
 } from "#recipes/sub/venue/sources"
+import { DEFAULT_VENUE_NAMES } from "#recipes/venue"
 import type { SubVenueLexiconTable } from "#subvenue/table"
 import type { LocaleBaseTuple } from "#surfaces/locale"
 import { alignRow } from "#utils"
 
-export * from "#recipes/sub/venue/sources"
-export * from "#recipes/sub/venue/context"
-export * from "#recipes/sub/venue/render"
-
 // #region Plan
 
 /**
@@ -62,7 +58,7 @@ export interface SubVenueLeg {
 	/**
 	 * The OSM extract filename under `--extracts-dir`.
 	 *
-	 * A leg without an extract draws its venue and confound pools from `poi.db`.
+	 * Every leg draws venue and confound pools from the Overture place-names file as well.
 	 */
 	extract?: string
 	/**
@@ -146,7 +142,7 @@ export const SUBVENUE_LEGS: readonly SubVenueLeg[] = [
  * The region whose identifier distribution the en-US leg uses.
  *
  * The en-US leg has no OSM extract.
- * Its `poi.db` contains venue names without refs.
+ * The Overture names file carries venue names without refs.
  * The recipe borrows the identifier distribution from GB.
  */
 export const US_IDENTIFIER_REGION_BORROWED_FROM = "GB"
@@ -636,23 +632,21 @@ export function allocate(total: number, shares: readonly number[]): number[] {
 }
 
 /**
- * Builds one leg's name pools from its extract and `poi.db`, plus its address context.
+ * Builds one leg's name pools from its extract and the Overture place-names file, plus its address context.
  */
 async function buildLegPools(
 	leg: SubVenueLeg,
 	query: PoolQuery,
 	contextByCountry: ReadonlyMap<string, LocaleBaseTuple[]>,
-	paths: { extractsDir: PathBuilderLike; poiDB: PathBuilderLike }
+	paths: { extractsDir: PathBuilderLike; venueNames: PathBuilderLike }
 ): Promise<LegPools> {
 	const extractPools = leg.extract
 		? await readExtractPools(`${paths.extractsDir}/${leg.extract}`, query)
 		: EMPTY_NAME_POOLS
 
-	// Only the US and FR legs have rows in poi.db.
-	const poiPools =
-		leg.country === "US" || leg.country === "FR" ? readPOIPools(paths.poiDB, leg.country, query) : EMPTY_NAME_POOLS
+	const overturePools = await readOvertureNamePools(paths.venueNames, leg.country, query)
 
-	const names = mergeNamePools(extractPools, poiPools)
+	const names = mergeNamePools(extractPools, overturePools)
 	let context = contextByCountry.get(leg.country) ?? []
 
 	if (leg.postcodePrefixes) {
@@ -711,8 +705,8 @@ export const subVenueRecipe: CorpusRecipe = {
 			description: "OSM sub-venue extract JSONLs (default: $MAILWOMAN_DATA_ROOT/sub-venue/extracts)",
 		},
 		{
-			flag: "--poi-db <path>",
-			description: "poi.db for the en-US / fr-FR pools (default: $MAILWOMAN_DATA_ROOT/db/poi/poi.db)",
+			flag: "--venue-names <parquet>",
+			description: `Overture place-names file for every leg's venue and confound pools (default ${DEFAULT_VENUE_NAMES})`,
 		},
 		{ flag: "--sub-venue-tuples <path>", description: "GB/US/FR address-context tuples JSONL" },
 		{
@@ -725,7 +719,7 @@ export const subVenueRecipe: CorpusRecipe = {
 		const count = opts.count
 		const negativeFraction = opts.negativeFraction ?? DEFAULT_NEGATIVE_FRACTION
 		const extractsDir = opts.extractsDir ?? dataRootPath("sub-venue", "extracts")
-		const poiDB = opts.poiDB ?? poiDatabasePath("poi.db")
+		const venueNames = opts.venueNames ?? DEFAULT_VENUE_NAMES
 		const tuplesPath = opts.subVenueTuples ?? dataRootPath("corpus", "intermediate", "house-venue-tuples-v3.jsonl")
 		const lexicon: SubVenueLexiconTable = await readSubVenueLexicon(opts.lexicon ?? defaultLexiconPath())
 
@@ -756,7 +750,7 @@ export const subVenueRecipe: CorpusRecipe = {
 				english: leg.english,
 			}
 
-			const pools = await buildLegPools(leg, query, contextByCountry, { extractsDir, poiDB })
+			const pools = await buildLegPools(leg, query, contextByCountry, { extractsDir, venueNames })
 
 			legPromoted.set(leg.locale, promoted)
 			legPools.set(leg.locale, pools)

package/lib/recipes/trailing-region.ts

@@ -42,8 +42,13 @@ import { SurfaceOrigin } from "#types"
 function regionCodeSurface(cc: string, region: string): string | null {
 	const country = cc.toUpperCase()
 
+	// country-branch: the codex holds region codes for CA and US only.
+	// The WOF admin database's `place_abbr` holds codes for ten countries
+	// (IT, US, ES, MX, BR, CA, AU, AE, UA, AD), and an ISO 3166-2 table would cover every country.
+	// The tree holds neither as a lookup yet.
 	if (country === "CA") return lookupCanadianProvince(region)
 
+	// country-branch: see the CA branch above.
 	return country === "US" ? lookupUSState(region) : null
 }
 
@@ -110,8 +115,7 @@ export const trailingRegionRecipe: CorpusRecipe = {
 			const placement = (t.postcodePlacement as PostcodePlacement | undefined) ?? "leading"
 			const bareLocality = postcode && placement === "after_locality" ? `${locality} ${postcode}` : locality
 
-			const localitySegment =
-				components["dependent_locality"] === undefined ? bareLocality : `${dependentLocality}, ${bareLocality}`
+			const localitySegment = components["dependent_locality"] ? `${dependentLocality}, ${bareLocality}` : bareLocality
 
 			// The code alternates with the region name rather than replacing it:
 			// both forms are posted and the resolver matches on the name.

package/lib/recipes/unit.ts

@@ -133,13 +133,13 @@ const tail = (random: () => number, loc: string, reg: string, pc: string): strin
 }
 
 /**
- * The identifier inside a rendered unit, or `undefined` when a standalone designator
+ * The identifier inside a rendered unit, or `null` when a standalone designator
  * or a letter-only id has none.
  */
-function unitIdentifier(unit: string): string | undefined {
+function unitIdentifier(unit: string): string | null {
 	const last = unit.replace(/^#\s*/, "").split(/\s+/).at(-1)
 
-	return last && /\d/.test(last) ? last : undefined
+	return last && /\d/.test(last) ? last : null
 }
 
 // `full-comma-bare` writes the identifier with no designator at all, the one unit surface carrying no