@mailwoman/corpus 10.0.0 → 10.1.0

package/lib/utils/wof-json.ts

@@ -3,34 +3,19 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters.
- *
- *   The Phase 1.5.1 pivot moved both adapters off the SpatiaLite distribution (dead mirror, empty
- *   `names` table) and onto the per-record GeoJSON bundles published as
- *   `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>` repos. Each repo carries
- *   a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
- *   `<id>-alt-quattroshapes.geojson`. The adapter only consumes the canonical record (no `-alt-`
- *   files); the alternate geometries are irrelevant to the name/hierarchy concerns Phase 1 cares
- *   about.
- *
- *   This module provides:
- *
- *   - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
- *   - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
- *       deprecated records).
- *   - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
- *   - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
- *   - `normalizeNameKey`: turns `"name:eng_x_colloquial"` into `"name-eng-x-colloquial"` for safe use
- *       in `source_id` suffixes.
- *
- *   `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live; `0` is
- *   superseded. WOF's official postalcode distribution stamps every row with `-1` ("unknown but
- *   treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
- *   silently emitted zero rows from the real corpus.
+ * Shared utilities for the `wof-admin` / `wof-postalcode` GeoJSON-bundle adapters. They read the
+ * per-record bundles published as `github.com/whosonfirst-data/whosonfirst-data-{admin,postalcode}-<cc>`
+ * repos: a tree of `data/<3>/<3>/<3>/<wof-id>.geojson` files plus alternate-geometry siblings like
+ * `<id>-alt-quattroshapes.geojson`, of which only the canonical record is consumed.
+ *
+ * `is_current` semantics follow WOF + Pelias: `mz:is_current` ∈ {`1`, `-1`} are live and `0` is
+ * superseded. WOF's official postalcode distribution stamps every row `-1` ("unknown but treated as
+ * active"), so an `is_current = 1` filter silently emits zero rows from the real corpus.
  */
 
 import { readLocalTextFile } from "@mailwoman/core/fs/readers"
 import { tryParsingJSON } from "@mailwoman/core/json"
+import type { PathBuilderLike } from "path-ts"
 import { Globerator } from "spliterator/node/fs"
 
 /**
@@ -43,37 +28,42 @@ export interface WOFFeature {
 }
 
 /**
- * Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped — it's 95% of the file
- * weight and the adapters never consult it.
+ * Lightweight in-memory shape both adapters keep per record.
+ *
+ * Geometry is intentionally dropped because the adapters never consult it.
  */
 export interface WOFRecord {
 	id: number
 	parent_id: number | null
-	/**
-	 * Canonical `wof:name` of the record.
-	 */
 	name: string
 	placetype: string
 	/**
-	 * ISO 3166-1 alpha-2 from `wof:country`.
+	 * ISO 3166-1 alpha-2 from `wof:country`, upper-cased on read.
+	 *
+	 * The standard's codes are upper case.
+	 * The publisher's spelling can differ.
+	 *
+	 * A consumer such as the `--country NL` filter compares the two strings directly.
 	 */
 	country: string
 	/**
 	 * Localized name variants from `name:*` properties.
 	 *
-	 * Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the underlying array (WOF
-	 * stores variants as arrays even when only one form is present). The canonical `wof:name` is NOT included here —
-	 * adapters add a synthetic `"default"` slot for it.
+	 * Keys are the raw `name:eng_x_preferred` form.
+	 * Values are the first non-empty string from the underlying array, because WOF
+	 * stores variants as arrays even when only one form is present.
+	 *
+	 * The canonical `wof:name` is not included — adapters add a synthetic `"default"` slot for it.
 	 */
 	nameVariants: Map<string, string>
 }
 
 /**
- * `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
+ * `mz:is_current` ∈ {`1`, `-1`} → keep, `0` → drop.
  *
- * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
- * alongside `1`. Tightening the predicate to `= 1` is the trap: the postalcode distros then contribute ZERO rows to the
- * corpus, silently, with nothing raised to notice it by.
+ * Real WOF postalcode distros tag every row `-1` ("unknown but treated as active") and the
+ * Pelias importer accepts `-1` alongside `1`; tightening the predicate to `= 1` makes the
+ * postalcode distros contribute zero rows to the corpus, silently, with no error to notice it by.
  */
 export function isCurrentFeature(props: Record<string, unknown>): boolean {
 	const raw = props["mz:is_current"]
@@ -83,9 +73,12 @@ export function isCurrentFeature(props: Record<string, unknown>): boolean {
 }
 
 /**
- * Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
- * Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
- * not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
+ * Pull `name:*` localized variants off a WOF feature's properties, lifting the first
+ * non-empty string from the array WOF stores them in (`["Saint Petersburg"]`);
+ * Multiple-value variants are rare and usually historical aliases.
+ *
+ * This function keeps them together.
+ * An adapter that needs separate values can iterate the underlying array.
  */
 export function extractNameVariants(props: Record<string, unknown>): Map<string, string> {
 	const out = new Map<string, string>()
@@ -93,32 +86,39 @@ export function extractNameVariants(props: Record<string, unknown>): Map<string,
 	for (const [key, value] of Object.entries(props)) {
 		if (!key.startsWith("name:")) continue
 
-		const candidate = Array.isArray(value)
-			? value.find((v): v is string => typeof v === "string" && v.trim().length > 0)
-			: typeof value === "string" && value.trim().length
-				? value
-				: undefined
+		const candidate = firstNonBlankString(value)
 
 		if (candidate) {
-			out.set(key, candidate.trim())
+			out.set(key, candidate)
 		}
 	}
 
 	return out
 }
 
+function firstNonBlankString(value: unknown): string | undefined {
+	for (const candidate of Array.isArray(value) ? value : [value]) {
+		if (typeof candidate !== "string") continue
+
+		const normalized = candidate.trim()
+
+		if (normalized) return normalized
+	}
+
+	return undefined
+}
+
 /**
- * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
- *
- * `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
- * existing source_id separator vocabulary and downstream consumers split on `-`.
+ * Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`
+ * (`"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`); both `:` and `_` become `-`
+ * because both collide with the separator vocabulary downstream consumers split on.
  */
 export function normalizeNameKey(rawKey: string): string {
 	return rawKey.replaceAll(/[:_]/g, "-")
 }
 
 /**
- * Result of parsing a single GeoJSON file. `null` means "skip this row" (any reason).
+ * Parse a single GeoJSON feature; `null` means skip this row for any reason.
  */
 function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 	if (!feature || feature.type !== "Feature" || !feature.properties) return null
@@ -137,9 +137,11 @@ function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 
 	if (typeof placetype !== "string" || !placetype) return null
 
-	const country = props["wof:country"]
+	const rawCountry = props["wof:country"]
 
-	if (typeof country !== "string" || !country) return null
+	if (typeof rawCountry !== "string" || !rawCountry) return null
+
+	const country = rawCountry.toUpperCase()
 
 	if (!isCurrentFeature(props)) return null
 
@@ -163,21 +165,25 @@ function recordFromFeature(feature: WOFFeature): WOFRecord | null {
 }
 
 /**
- * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
+ * Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s;
+ * `repoDir` may be a single cloned `whosonfirst-data-*` repo or a parent holding several
+ * (the corpus pipeline clones all four into a shared `wof/repos/` root).
  *
- * `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
- * (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
- * `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
- * records.
+ * `-alt-` siblings are skipped as alternate-geometry exports.
+ * They do not represent new records.
  *
- * Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
- * walk. Adapters can add stricter validation downstream if they need it.
+ * Per-file errors (unreadable files, malformed JSON, or missing properties) are swallowed
+ * so one bad file does not poison a multi-gigabyte walk.
  */
-export async function* walkFeatures(repoDir: string, opts: { signal?: AbortSignal } = {}): AsyncIterable<WOFRecord> {
+export async function* walkFeatures(
+	repoDir: PathBuilderLike,
+	opts: { signal?: AbortSignal } = {}
+): AsyncIterable<WOFRecord> {
 	for await (const filePath of Globerator.from("**/*.geojson", {
 		cwd: repoDir,
+		absolute: true,
 		exclude: ["**/*-alt-*.geojson"],
-		// Preserve the recursive fast-glob walk this replaces: bundle repositories can expose data through a link.
+		// Bundle repositories can expose their data through a link.
 		followSymlinks: true,
 		signal: opts.signal,
 	})) {
@@ -202,24 +208,63 @@ export async function* walkFeatures(repoDir: string, opts: { signal?: AbortSigna
 }
 
 /**
- * Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id` upward and stopping
- * at the first missing link. A cycle guard halts at any re-visit (defensive — WOF data is acyclic by construction but
- * corrupt fixtures shouldn't infinite-loop the adapter).
+ * The component tags a variant builder looks an ancestor up by.
+ *
+ * Measured over both adapters on 2026-09-28: `variantsFor` reads the nearest `region`
+ * and `country` ancestor, `postcodeVariantsFor` reads the nearest `locality`, `region` and `country`,
+ * and each one reads the ancestor's `placetype` to classify it and its `name` to use it.
+ * No caller reads an ancestor's `nameVariants`, `id`, `parent_id` or `country`.
+ */
+export const ANCESTOR_NAME_TAGS = ["locality", "region", "country"] as const
+
+/**
+ * The nearest ancestor's `wof:name` per tag in {@linkcode ANCESTOR_NAME_TAGS}.
+ *
+ * A tag with no ancestor has no key.
+ * `ancestry.find(…) === undefined` expresses that state.
+ */
+export type AncestorNames = Partial<Record<(typeof ANCESTOR_NAME_TAGS)[number], string>>
+
+/**
+ * The ancestor names each record needs, keyed by `wof:id`.
+ *
+ * The predecessor index held one array of record references per record, so a run over the 13,274,691
+ * admin records of `v0.7.0-de-holdout` allocated 13.3 million arrays alongside the record index.
+ * This holds at most three strings per record instead.
+ *
+ * Each string is the same object the ancestor already holds, so the index costs
+ * one small object per record without copying a name.
  *
- * Records whose ancestors aren't in `byID` (e.g. an FR locality whose region wasn't included in the cloned repo set)
- * get a shorter chain; the variant emission gracefully degrades.
+ * The adapter's second pass reads only this.
+ * `wof-postalcode` keeps admin records in its record index solely so postcode ancestry resolves.
+ * It can release them once this index is built.
  *
- * `wof-admin-jp`'s `chainOf` walks the same parent-child relation over SQLite `spr` rows with an
- * outside-the-preloaded-set fallback query — a different substrate with a different escape hatch, kept separate on
- * purpose.
+ * Nearest wins: the walk goes upward from the record's parent and the first ancestor
+ * carrying a tag sets it, matching the `ancestry.find` the builders used.
  */
-export type AncestryIndex = Map<number, WOFRecord[]>
+export type AncestorNameIndex = Map<number, AncestorNames>
 
-export function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex {
-	const index: AncestryIndex = new Map()
+/**
+ * Resolves {@linkcode AncestorNameIndex} from a record index.
+ *
+ * `tagOf` maps a WOF placetype to a component tag, or returns a value outside
+ * {@linkcode ANCESTOR_NAME_TAGS} for a placetype no builder looks up.
+ * It is a parameter because `placetypeToTag` lives with the adapters and this module does not import them.
+ *
+ * The walk stops at the first `parent_id` absent from `byID`, and a cycle guard halts at any re-visit.
+ * WOF data is acyclic by construction.
+ * The guard keeps a corrupt fixture from looping the adapter.
+ *
+ * A record whose ancestors are absent resolves the tags it can reach rather than failing.
+ */
+export function buildAncestorNameIndex(
+	byID: ReadonlyMap<number, WOFRecord>,
+	tagOf: (placetype: string) => string | null | undefined
+): AncestorNameIndex {
+	const index: AncestorNameIndex = new Map()
 
 	for (const [id, rec] of byID) {
-		const chain: WOFRecord[] = []
+		const names: AncestorNames = {}
 		const guard = new Set<number>([id])
 		let cur: number | null = rec.parent_id
 
@@ -229,12 +274,20 @@ export function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex
 			if (!parent) break
 
 			if (guard.has(parent.id)) break
-			chain.push(parent)
+
+			const tag = tagOf(parent.placetype)
+
+			for (const wanted of ANCESTOR_NAME_TAGS) {
+				if (tag === wanted && names[wanted] === undefined) {
+					names[wanted] = parent.name
+				}
+			}
+
 			guard.add(parent.id)
 			cur = parent.parent_id
 		}
 
-		index.set(id, chain)
+		index.set(id, names)
 	}
 
 	return index

package/lib/{utils/index.ts → utils.ts}

@@ -7,7 +7,6 @@
 export * from "#utils/align"
 export * from "#build"
 export * from "#utils/golden"
-export * from "#utils/parquet"
 export * from "#utils/license"
 export * from "#utils/split"
 export * from "#utils/tokenize"

package/lib/ve/recipes/reviewed-postcode-tail.ts

@@ -3,25 +3,26 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   A bounded, reviewed source for Venezuela's `locality postcode, region` convention (#1821).
- *   Bulk sources yielded zero Venezuelan tuples, so this recipe reads four reviewed facts and applies
- *   only transformations that do not create another postcode-to-place join.
+ *   Renders four hand-reviewed Venezuelan postcode facts in the `locality postcode, region` order.
  */
 
 import { readLocalJSONFile } from "@mailwoman/core/fs/readers"
 import { resolveModulePath } from "@mailwoman/core/module/resolvers"
 import { stripCombiningMarks } from "@mailwoman/normalize/fold"
+import type { PathBuilderLike } from "path-ts"
 
-import { alignAndWrite, type CorpusRecipe, sliceSourceID } from "#recipes/scaffold"
+import { alignAndWrite, type CorpusRecipe, recipeSourceID } from "#recipes/scaffold"
+import { defaultRecipeSource } from "#recipes/sources"
+import { SourceRegister } from "#registers"
+import { SurfaceOrigin } from "#types"
 
 /**
- * A separate sampler bucket so a receipt measures these reviewed after-locality rows and no other postcode placement.
+ * The source label for these rows.
+ *
+ * The rows get their own sampler bucket so an eval can measure them apart from other postcode placements.
  */
-export const REVIEWED_POSTCODE_TAIL_SOURCE = "synth-reviewed-postcode-tail"
+export const REVIEWED_POSTCODE_TAIL_SOURCE = defaultRecipeSource("synth-reviewed-postcode-tail")
 
-/**
- * The four postcode-to-place facts reviewed for #1821 and committed in the package data file.
- */
 const REVIEWED_TUPLE_COUNT = 4
 
 interface ReviewedTupleProvenance {
@@ -32,6 +33,9 @@ interface ReviewedTupleProvenance {
 	sourceLicenseNote: string
 }
 
+/**
+ * One reviewed postcode fact and the source it was read from.
+ */
 export interface ReviewedPostcodeTuple {
 	id: string
 	locality: string
@@ -66,12 +70,20 @@ interface Variant {
 
 const ISO_DATE = /^\d{4}-\d{2}-\d{2}$/
 
+/**
+ * Returns the path of the committed reviewed-tuple data file.
+ */
 export function defaultReviewedPostcodeTuplePath(): string {
 	return resolveModulePath("@mailwoman/corpus/data/reviewed-ve-postcode-tuples.json")
 }
 
+/**
+ * Reads and validates the reviewed tuples.
+ *
+ * @throws When the file does not hold exactly four complete, unique, reviewed tuples.
+ */
 export async function readReviewedPostcodeTuples(
-	path = defaultReviewedPostcodeTuplePath()
+	path: PathBuilderLike = defaultReviewedPostcodeTuplePath()
 ): Promise<ReviewedPostcodeTuple[]> {
 	const document = await readLocalJSONFile<ReviewedPostcodeTupleFile>(path)
 
@@ -114,6 +126,12 @@ export async function readReviewedPostcodeTuples(
 
 const foldAccents = (value: string): string => stripCombiningMarks(value).normalize("NFC")
 
+/**
+ * Returns the surface variants of one tuple.
+ *
+ * Every variant reuses the tuple's own locality, postcode and region, so no variant
+ * pairs a postcode with a place the review did not cover.
+ */
 export function reviewedPostcodeTailVariants(tuple: ReviewedPostcodeTuple): Variant[] {
 	const tail = `${tuple.locality} ${tuple.postcode}, ${tuple.region}`
 	const withCountry = `${tail}, ${tuple.country}`
@@ -151,7 +169,7 @@ export function reviewedPostcodeTailVariants(tuple: ReviewedPostcodeTuple): Vari
 }
 
 /**
- * Emit bounded surface variants of the committed reviewed facts without constructing another geographic join.
+ * The corpus recipe that emits every variant of every reviewed tuple.
  */
 export const reviewedPostcodeTailRecipe: CorpusRecipe = {
 	name: "reviewed-postcode-tail",
@@ -164,7 +182,7 @@ export const reviewedPostcodeTailRecipe: CorpusRecipe = {
 
 		for (const tuple of tuples) {
 			for (const variant of reviewedPostcodeTailVariants(tuple)) {
-				const sourceID = sliceSourceID(REVIEWED_POSTCODE_TAIL_SOURCE, { tuple: tuple.id, variant: variant.id })
+				const sourceID = recipeSourceID(REVIEWED_POSTCODE_TAIL_SOURCE, { tuple: tuple.id, variant: variant.id })
 
 				const canonical = {
 					raw: variant.raw,
@@ -177,7 +195,16 @@ export const reviewedPostcodeTailRecipe: CorpusRecipe = {
 					license: `Synthetic rendering of reviewed postal facts — ${tuple.provenance.publisher}; source terms recorded in reviewed-ve-postcode-tuples.json`,
 				}
 
-				if (alignAndWrite(write, canonical, "reviewed-postcode-tail", tuple.id)) {
+				if (
+					alignAndWrite(write, canonical, "reviewed-postcode-tail", {
+						// A person read these facts from their publishers.
+						// The data file records the publisher of each tuple.
+						// No upstream bulk file exists.
+						register: SourceRegister.ReviewedByHand,
+						surface: SurfaceOrigin.Composed,
+						baseSourceID: tuple.id,
+					})
+				) {
 					emitted++
 				} else {
 					skipped++