@mailwoman/corpus 9.4.0 → 10.1.0

package/lib/utils/wof-json.ts

@@ -3,35 +3,20 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
- *
- *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
- *   `names` table) and onto the per-record GeoJSON bundles published as
- *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
- *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
- *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
- *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
- *   about.
- *
- *   This module provides:
- *
- *   - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
- *   - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
- *       deprecated records).
- *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
- *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
- *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
- *       in `source_id` suffixes.
- *
- *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
- *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
- *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
- *   silently emitted zero rows from the real corpus.
+ * Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters. They read the
+ * per-record bundles published as `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>`
+ * repos: a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
+ * `<id>-alt-quattroshapes.geojson`, of which only the canonical record is consumed.
+ *
+ * `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live and `0` is
+ * superseded. WOF's official postalcode distribution stamps every row `-1` ("unknown but treated as
+ * active"), so an `is_current = 1` filter silently emits zero rows from the real corpus.
  */
 
 import { readLocalTextFile } from "@mailwoman/core/fs/readers"
 import { tryParsingJSON } from "@mailwoman/core/json"
-import FastGlob from "fast-glob"
+import type { PathBuilderLike } from "path-ts"
+import { Globerator } from "spliterator/node/fs"
 
 /**
  * A WOF GeoJSON feature, as published by the per-record bundles.
@@ -43,37 +28,42 @@ export interface WOFFeature {
 }
 
 /**
- * Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped — it's 95% of the file
- * weight and the adapters never consult it.
+ * Lightweight in-memory shape both adapters keep per record.
+ *
+ * Geometry is intentionally dropped because the adapters never consult it.
  */
 export interface WOFRecord {
 	id: number
 	parent_id: number | null
-	/**
-	 * Canonical `wof:name` of the record.
-	 */
 	name: string
 	placetype: string
 	/**
-	 * ISO 3166-1 alpha-2 from `wof:country`.
+	 * ISO 3166-1 alpha-2 from `wof:country`, upper-cased on read.
+	 *
+	 * The standard's codes are upper case.
+	 * The publisher's spelling can differ.
+	 *
+	 * A consumer such as the `--country NL` filter compares the two strings directly.
 	 */
 	country: string
 	/**
 	 * Localized name variants from `name:*` properties.
 	 *
-	 * Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the underlying array (WOF
-	 * stores variants as arrays even when only one form is present). The canonical `wof:name` is NOT included here —
-	 * adapters add a synthetic `"default"` slot for it.
+	 * Keys are the raw `name:eng_x_preferred` form.
+	 * Values are the first non-empty string from the underlying array, because WOF
+	 * stores variants as arrays even when only one form is present.
+	 *
+	 * The canonical `wof:name` is not included — adapters add a synthetic `"default"` slot for it.
 	 */
 	nameVariants: Map<string, string>
 }
 
 /**
- * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
+ * `mz:is_current` ∈ {`1`, `-1`} → keep, `0` → drop.
  *
- * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
- * alongside `1`. Tightening the predicate to `= 1` is the trap: the postalcode distros then contribute ZERO rows to the
- * corpus, silently, with nothing raised to notice it by.
+ * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active") and the
+ * Pelias importer accepts `-1` alongside `1`; tightening the predicate to `= 1` makes the
+ * postalcode distros contribute zero rows to the corpus, silently, with no error to notice it by.
  */
 export function isCurrentFeature(props: Record<string, unknown>): boolean {
 	const raw = props["mz:is_current"]
@@ -83,9 +73,12 @@ export function isCurrentFeature(props: Record<string, unknown>): boolean {
 }
 
 /**
- * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
- * Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
- * not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
+ * Pull `name:*` localized variants off a WOF feature's properties, lifting the first
+ * non-empty string from the array WOF stores them in (`["Saint Petersburg"]`);
+ * Multiple-value variants are rare and usually historical aliases.
+ *
+ * This function keeps them together.
+ * An adapter that needs separate values can iterate the underlying array.
  */
 export function extractNameVariants(props: Record<string, unknown>): Map<string, string> {
 	const out = new Map<string, string>()
@@ -93,32 +86,39 @@ export function extractNameVariants(props: Record<string, unknown>): Map<string,
 	for (const [key, value] of Object.entries(props)) {
 		if (!key.startsWith("name:")) continue
 
-		const candidate = Array.isArray(value)
-			? value.find((v): v is string => typeof v === "string" && v.trim().length > 0)
-			: typeof value === "string" && value.trim().length
-				? value
-				: undefined
+		const candidate = firstNonBlankString(value)
 
 		if (candidate) {
-			out.set(key, candidate.trim())
+			out.set(key, candidate)
 		}
 	}
 
 	return out
 }
 
+function firstNonBlankString(value: unknown): string | undefined {
+	for (const candidate of Array.isArray(value) ? value : [value]) {
+		if (typeof candidate !== "string") continue
+
+		const normalized = candidate.trim()
+
+		if (normalized) return normalized
+	}
+
+	return undefined
+}
+
 /**
- * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
- *
- * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
- * existing source_id separator vocabulary and downstream consumers split on `-`.
+ * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`
+ * (`"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`); both `:` and `_` become `-`
+ * because both collide with the separator vocabulary downstream consumers split on.
  */
 export function normalizeNameKey(rawKey: string): string {
 	return rawKey.replaceAll(/[:_]/g, "-")
 }
 
 /**
- * Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason).
+ * Parse a single GeoJSON feature; `null` means skip this row for any reason.
  */
 function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 	if (!feature || feature.type !== "Feature" || !feature.properties) return null
@@ -137,9 +137,11 @@ function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 
 	if (typeof placetype !== "string" || !placetype) return null
 
-	const country = props["wof:country"]
+	const rawCountry = props["wof:country"]
 
-	if (typeof country !== "string" || !country) return null
+	if (typeof rawCountry !== "string" || !rawCountry) return null
+
+	const country = rawCountry.toUpperCase()
 
 	if (!isCurrentFeature(props)) return null
 
@@ -163,30 +165,28 @@ function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 }
 
 /**
- * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
+ * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s;
+ * `repoDir` may be a single cloned `whosonfirst-data-*` repo or a parent holding several
+ * (the corpus pipeline clones all four into a shared `wof/repos/` root).
  *
- * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
- * (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
- * `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
- * records.
+ * `-alt-` siblings are skipped as alternate-geometry exports.
+ * They do not represent new records.
  *
- * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
- * walk. Adapters can add stricter validation downstream if they need it.
+ * Per-file errors (unreadable files, malformed JSON, or missing properties) are swallowed
+ * so one bad file does not poison a multi-gigabyte walk.
  */
-export async function* walkFeatures(repoDir: string, opts: { signal?: AbortSignal } = {}): AsyncIterable<WOFRecord> {
-	const stream = FastGlob.stream(["**/*.geojson"], {
+export async function* walkFeatures(
+	repoDir: PathBuilderLike,
+	opts: { signal?: AbortSignal } = {}
+): AsyncIterable<WOFRecord> {
+	for await (const filePath of Globerator.from("**/*.geojson", {
 		cwd: repoDir,
 		absolute: true,
-		onlyFiles: true,
-		suppressErrors: true,
-	})
-
-	for await (const entry of stream) {
-		if (opts.signal?.aborted) return
-		const filePath = String(entry)
-
-		if (filePath.includes("-alt-")) continue
-
+		exclude: ["**/*-alt-*.geojson"],
+		// Bundle repositories can expose their data through a link.
+		followSymlinks: true,
+		signal: opts.signal,
+	})) {
 		let text: string
 
 		try {
@@ -208,24 +208,63 @@ export async function* walkFeatures(repoDir: string, opts: { signal?: AbortSigna
 }
 
 /**
- * Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id` upward and stopping
- * at the first missing link. A cycle guard halts at any re-visit (defensive — WOF data is acyclic by construction but
- * corrupt fixtures shouldn't infinite-loop the adapter).
+ * The component tags a variant builder looks an ancestor up by.
  *
- * Records whose ancestors aren't in `byID` (e.g. an FR locality whose region wasn't included in the cloned repo set)
- * get a shorter chain; the variant emission gracefully degrades.
+ * Measured over both adapters on 2026-09-28: `variantsFor` reads the nearest `region`
+ * and `country` ancestor, `postcodeVariantsFor` reads the nearest `locality`, `region` and `country`,
+ * and each one reads the ancestor's `placetype` to classify it and its `name` to use it.
+ * No caller reads an ancestor's `nameVariants`, `id`, `parent_id` or `country`.
+ */
+export const ANCESTOR_NAME_TAGS = ["locality", "region", "country"] as const
+
+/**
+ * The nearest ancestor's `wof:name` per tag in {@linkcode ANCESTOR_NAME_TAGS}.
+ *
+ * A tag with no ancestor has no key.
+ * `ancestry.find(…) === undefined` expresses that state.
+ */
+export type AncestorNames = Partial<Record<(typeof ANCESTOR_NAME_TAGS)[number], string>>
+
+/**
+ * The ancestor names each record needs, keyed by `wof:id`.
+ *
+ * The predecessor index held one array of record references per record, so a run over the 13,274,691
+ * admin records of `v0.7.0-de-holdout` allocated 13.3 million arrays alongside the record index.
+ * This holds at most three strings per record instead.
+ *
+ * Each string is the same object the ancestor already holds, so the index costs
+ * one small object per record without copying a name.
  *
- * `wof-admin-jp`'s `chainOf` walks the same parent-child relation over SQLite `spr` rows with an
- * outside-the-preloaded-set fallback query — a different substrate with a different escape hatch, kept separate on
- * purpose.
+ * The adapter's second pass reads only this.
+ * `wof-postalcode` keeps admin records in its record index solely so postcode ancestry resolves.
+ * It can release them once this index is built.
+ *
+ * Nearest wins: the walk goes upward from the record's parent and the first ancestor
+ * carrying a tag sets it, matching the `ancestry.find` the builders used.
  */
-export type AncestryIndex = Map<number, WOFRecord[]>
+export type AncestorNameIndex = Map<number, AncestorNames>
 
-export function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex {
-	const index: AncestryIndex = new Map()
+/**
+ * Resolves {@linkcode AncestorNameIndex} from a record index.
+ *
+ * `tagOf` maps a WOF placetype to a component tag, or returns a value outside
+ * {@linkcode ANCESTOR_NAME_TAGS} for a placetype no builder looks up.
+ * It is a parameter because `placetypeToTag` lives with the adapters and this module does not import them.
+ *
+ * The walk stops at the first `parent_id` absent from `byID`, and a cycle guard halts at any re-visit.
+ * WOF data is acyclic by construction.
+ * The guard keeps a corrupt fixture from looping the adapter.
+ *
+ * A record whose ancestors are absent resolves the tags it can reach rather than failing.
+ */
+export function buildAncestorNameIndex(
+	byID: ReadonlyMap<number, WOFRecord>,
+	tagOf: (placetype: string) => string | null | undefined
+): AncestorNameIndex {
+	const index: AncestorNameIndex = new Map()
 
 	for (const [id, rec] of byID) {
-		const chain: WOFRecord[] = []
+		const names: AncestorNames = {}
 		const guard = new Set<number>([id])
 		let cur: number | null = rec.parent_id
 
@@ -235,12 +274,20 @@ export function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex
 			if (!parent) break
 
 			if (guard.has(parent.id)) break
-			chain.push(parent)
+
+			const tag = tagOf(parent.placetype)
+
+			for (const wanted of ANCESTOR_NAME_TAGS) {
+				if (tag === wanted && names[wanted] === undefined) {
+					names[wanted] = parent.name
+				}
+			}
+
 			guard.add(parent.id)
 			cur = parent.parent_id
 		}
 
-		index.set(id, chain)
+		index.set(id, names)
 	}
 
 	return index

package/lib/utils.ts

@@ -0,0 +1,13 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+export * from "#utils/align"
+export * from "#build"
+export * from "#utils/golden"
+export * from "#utils/license"
+export * from "#utils/split"
+export * from "#utils/tokenize"
+export * from "#utils/wof-json"

package/lib/ve/recipes/reviewed-postcode-tail.ts

@@ -0,0 +1,217 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Renders four hand-reviewed Venezuelan postcode facts in the `locality postcode, region` order.
+ */
+
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
+import { resolveModulePath } from "@mailwoman/core/module/resolvers"
+import { stripCombiningMarks } from "@mailwoman/normalize/fold"
+import type { PathBuilderLike } from "path-ts"
+
+import { alignAndWrite, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { SurfaceOrigin } from "#types"
+
+/**
+ * The source label for these rows.
+ *
+ * The rows get their own sampler bucket so an eval can measure them apart from other postcode placements.
+ */
+export const REVIEWED_POSTCODE_TAIL_SOURCE = defaultRecipeSource("synth-reviewed-postcode-tail")
+
+const REVIEWED_TUPLE_COUNT = 4
+
+interface ReviewedTupleProvenance {
+	publisher: string
+	url: string
+	retrievedAt: string
+	reviewStatus: "reviewed"
+	sourceLicenseNote: string
+}
+
+/**
+ * One reviewed postcode fact and the source it was read from.
+ */
+export interface ReviewedPostcodeTuple {
+	id: string
+	locality: string
+	postcode: string
+	region: string
+	country: "Venezuela"
+	cc: "VE"
+	locale: "es-VE"
+	postcodePlacement: "after_locality"
+	provenance: ReviewedTupleProvenance
+}
+
+interface ReviewedPostcodeTupleFile {
+	version: string
+	reviewedAt: string
+	tuples: ReviewedPostcodeTuple[]
+}
+
+type VariantID =
+	| "canonical-country"
+	| "canonical-no-country"
+	| "comma-free"
+	| "uppercase"
+	| "left-context"
+	| "accent-folded"
+
+interface Variant {
+	id: VariantID
+	raw: string
+	components: Record<string, string>
+}
+
+const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
+
+/**
+ * Returns the path of the committed reviewed-tuple data file.
+ */
+export function defaultReviewedPostcodeTuplePath(): string {
+	return resolveModulePath("@mailwoman/corpus/data/reviewed-ve-postcode-tuples.json")
+}
+
+/**
+ * Reads and validates the reviewed tuples.
+ *
+ * @throws When the file does not hold exactly four complete, unique, reviewed tuples.
+ */
+export async function readReviewedPostcodeTuples(
+	path: PathBuilderLike = defaultReviewedPostcodeTuplePath()
+): Promise<ReviewedPostcodeTuple[]> {
+	const document = await readLocalJSONFile<ReviewedPostcodeTupleFile>(path)
+
+	if (!document.version || !ISO_DATE.test(document.reviewedAt) || document.tuples.length !== REVIEWED_TUPLE_COUNT) {
+		throw new Error("reviewed postcode tuple file must declare a version, review date, and exactly four tuples")
+	}
+
+	const ids = new Set<string>()
+	const facts = new Set<string>()
+
+	for (const tuple of document.tuples) {
+		const provenance = tuple.provenance
+		const fact = `${tuple.locality}\u0000${tuple.postcode}\u0000${tuple.region}`
+
+		if (
+			!tuple.id ||
+			!tuple.locality ||
+			!tuple.postcode ||
+			!tuple.region ||
+			tuple.country !== "Venezuela" ||
+			tuple.cc !== "VE" ||
+			tuple.locale !== "es-VE" ||
+			tuple.postcodePlacement !== "after_locality" ||
+			!provenance?.publisher ||
+			!URL.canParse(provenance.url) ||
+			!ISO_DATE.test(provenance.retrievedAt) ||
+			provenance.reviewStatus !== "reviewed" ||
+			!provenance.sourceLicenseNote
+		) {
+			throw new Error(`invalid reviewed postcode tuple: ${tuple.id || "<missing id>"}`)
+		}
+
+		if (ids.has(tuple.id) || facts.has(fact)) throw new Error(`duplicate reviewed postcode tuple: ${tuple.id}`)
+		ids.add(tuple.id)
+		facts.add(fact)
+	}
+
+	return document.tuples
+}
+
+const foldAccents = (value: string): string => stripCombiningMarks(value).normalize("NFC")
+
+/**
+ * Returns the surface variants of one tuple.
+ *
+ * Every variant reuses the tuple's own locality, postcode and region, so no variant
+ * pairs a postcode with a place the review did not cover.
+ */
+export function reviewedPostcodeTailVariants(tuple: ReviewedPostcodeTuple): Variant[] {
+	const tail = `${tuple.locality} ${tuple.postcode}, ${tuple.region}`
+	const withCountry = `${tail}, ${tuple.country}`
+	const base = { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region, country: tuple.country }
+	const upper = Object.fromEntries(Object.entries(base).map(([key, value]) => [key, value.toUpperCase()]))
+
+	const variants: Variant[] = [
+		{ id: "canonical-country", raw: withCountry, components: base },
+		{
+			id: "canonical-no-country",
+			raw: tail,
+			components: { locality: tuple.locality, postcode: tuple.postcode, region: tuple.region },
+		},
+		{ id: "comma-free", raw: `${tuple.locality} ${tuple.postcode} ${tuple.region} ${tuple.country}`, components: base },
+		{ id: "uppercase", raw: withCountry.toUpperCase(), components: upper },
+		{
+			id: "left-context",
+			raw: `Comercio Ejemplo, Calle Principal, ${withCountry}`,
+			components: { venue: "Comercio Ejemplo", street: "Calle Principal", ...base },
+		},
+	]
+
+	const foldedLocality = foldAccents(tuple.locality)
+	const foldedRegion = foldAccents(tuple.region)
+
+	if (foldedLocality !== tuple.locality || foldedRegion !== tuple.region) {
+		variants.push({
+			id: "accent-folded",
+			raw: `${foldedLocality} ${tuple.postcode}, ${foldedRegion}, ${tuple.country}`,
+			components: { locality: foldedLocality, postcode: tuple.postcode, region: foldedRegion, country: tuple.country },
+		})
+	}
+
+	return variants
+}
+
+/**
+ * The corpus recipe that emits every variant of every reviewed tuple.
+ */
+export const reviewedPostcodeTailRecipe: CorpusRecipe = {
+	name: "reviewed-postcode-tail",
+	description: "Reviewed Venezuela locality-postcode-region tails with bounded surface variants",
+	mode: "generate",
+	async run(opts, write) {
+		const tuples = await readReviewedPostcodeTuples(opts.input)
+		let emitted = 0
+		let skipped = 0
+
+		for (const tuple of tuples) {
+			for (const variant of reviewedPostcodeTailVariants(tuple)) {
+				const sourceID = recipeSourceID(REVIEWED_POSTCODE_TAIL_SOURCE, { tuple: tuple.id, variant: variant.id })
+
+				const canonical = {
+					raw: variant.raw,
+					components: variant.components,
+					country: tuple.cc,
+					locale: tuple.locale,
+					source: REVIEWED_POSTCODE_TAIL_SOURCE,
+					source_id: sourceID,
+					corpus_version: "0.11.0",
+					license: `Synthetic rendering of reviewed postal facts — ${tuple.provenance.publisher}; source terms recorded in reviewed-ve-postcode-tuples.json`,
+				}
+
+				if (
+					alignAndWrite(write, canonical, "reviewed-postcode-tail", {
+						// A person read these facts from their publishers.
+						// The data file records the publisher of each tuple.
+						// No upstream bulk file exists.
+						register: SourceRegister.ReviewedByHand,
+						surface: SurfaceOrigin.Composed,
+						baseSourceID: tuple.id,
+					})
+				) {
+					emitted++
+				} else {
+					skipped++
+				}
+			}
+		}
+
+		return { read: tuples.length, emitted, skipped }
+	},
+}