mailwoman 8.5.0 → 8.6.0

package/commands/gazetteer/granularity.tsx

@@ -0,0 +1,104 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   `mailwoman gazetteer granularity` — the per-country gazetteer depth scorecard.
+ *
+ *   Answers "where does the admin gazetteer bottom out?" for every country it knows, which nobody
+ *   had measured: the shipped artifact stocks 9 of WOF's 34 placetypes and carries a
+ *   `dependent_locality` tier in 11 of 244 countries. Read-only, no network, no model — three
+ *   grouped queries over `spr`/`ancestors` and a markdown render.
+ *
+ *   The report is a COMMITTED artifact (the `fill-rates.md` precedent), so the source md5 and the
+ *   parent-coverage floor are pinned in its header. Re-running against a rebuilt gazetteer and
+ *   diffing the report is the intended workflow.
+ */
+
+import { mkdirSync, writeFileSync } from "node:fs"
+import { dirname } from "node:path"
+
+import { dataRootPath, md5File } from "@mailwoman/core/utils"
+import { Box, Text } from "ink"
+import zod from "zod"
+
+import { type CommandComponent, useCommandTask } from "../../cli-kit/index.ts"
+import { renderGranularityReport } from "../../gazetteer-pipeline/granularity-report.ts"
+import { DEFAULT_COVERAGE_FLOOR, bottomsOutAt, buildGranularityLadder } from "../../gazetteer-pipeline/granularity.ts"
+
+const OptionsSchema = zod.object({
+	out: zod
+		.string()
+		.default("docs/articles/evals/coverage/gazetteer-depth-scorecard.md")
+		.describe("Output path for the markdown scorecard"),
+	source: zod.string().optional().describe("WOF admin DB. Default $MAILWOMAN_DATA_ROOT/wof/admin-global-priority.db"),
+	floor: zod
+		.number()
+		.default(DEFAULT_COVERAGE_FLOOR)
+		.describe(
+			`Parent-coverage floor for crediting a sub-locality rung. Default ${DEFAULT_COVERAGE_FLOOR} — set low on ` +
+				"purpose, to catch thin-but-real tiers rather than certify them."
+		),
+})
+
+export { OptionsSchema as options }
+
+const GazetteerGranularity: CommandComponent<typeof OptionsSchema> = ({ options }) => {
+	const state = useCommandTask(async () => {
+		const sourcePath = options.source ?? String(dataRootPath("wof", "admin-global-priority.db"))
+		const rows = buildGranularityLadder(sourcePath)
+
+		if (!rows.length) {
+			throw new Error(`granularity: no countries measured from ${sourcePath} — is this an admin DB?`)
+		}
+
+		const markdown = renderGranularityReport(rows, {
+			// Display the portable form; never bake the resolved lab path into a committed artifact.
+			sourcePath: "$MAILWOMAN_DATA_ROOT/wof/admin-global-priority.db",
+			sourceMD5: await md5File(sourcePath),
+			buildDate: new Date().toISOString(),
+			floor: options.floor,
+		})
+
+		mkdirSync(dirname(options.out), { recursive: true })
+		writeFileSync(options.out, markdown)
+
+		const byBottom = new Map<string, number>()
+
+		for (const row of rows) {
+			const bottom = bottomsOutAt(row, options.floor) ?? "(none)"
+
+			byBottom.set(bottom, (byBottom.get(bottom) ?? 0) + 1)
+		}
+
+		const distribution = [...byBottom.entries()]
+			.toSorted((a, b) => b[1] - a[1])
+			.map(([rung, n]) => `  ${rung}: ${n.toLocaleString()} countries`)
+
+		return [
+			`gazetteer depth scorecard → ${options.out}`,
+			`countries measured: ${rows.length.toLocaleString()} (floor ${(options.floor * 100).toFixed(1)}%)`,
+			"bottoms out at:",
+			...distribution,
+		]
+	})
+
+	if (state.status === "error") return <Text color="red">✗ {state.message}</Text>
+
+	if (state.status === "done") {
+		return (
+			<Box flexDirection="column">
+				{state.result.map((line, i) => (
+					<Text key={i} color={i === 0 ? "green" : undefined}>
+						{i === 0 ? "✓ " : "  "}
+						{line}
+					</Text>
+				))}
+			</Box>
+		)
+	}
+
+	return null
+}
+
+export default GazetteerGranularity

package/commands/gazetteer/pair-index.tsx

@@ -70,7 +70,7 @@ const RUNG3_PRE_FOLD_CENSUS_LINE_COUNT = 19_431
  * tracks the WOF snapshot alone. A snapshot refresh legitimately moves it; re-anchor the constant deliberately, after
  * inspecting the diff, rather than relaxing the gate.
  */
-const EXPECTED_US_PAIR_COUNT = 49_033
+const EXPECTED_US_PAIR_COUNT = 47_878
 
 /**
  * Known (child, parent) pairs probed after write as a self-check — PER COUNTRY, keyed by the `--country` code. Probing
@@ -97,6 +97,27 @@ const PROBE_PAIRS_BY_COUNTRY: Readonly<Record<string, ReadonlyArray<readonly [ci
 		["Park Slope", "Brooklyn"],
 		["Manhattan", "New York"],
 	],
+	// R10: the IN instance. Indian addresses carry an area/locality line above the city
+	// ("Indiranagar, Bengaluru"), which is the dependent-locality slot.
+	in: [
+		["Indiranagar", "Bangalore"],
+		["Mulund East", "Mumbai"],
+	],
+	// R11: the ES + IT instances. Spanish barrios and Italian quartieri under their municipio/comune.
+	es: [
+		["Aravaca", "Madrid"],
+		["Triana", "Sevilla"],
+	],
+	it: [
+		["Barona", "Milano"],
+		["Trastevere", "Roma"],
+	],
+	// R9: the DE instance. Ortsteile/Stadtteile under their Gemeinde — the line German addresses
+	// carry when they carry one at all (the Ortsteil sits above "PLZ Stadt", never inside it).
+	de: [
+		["Nippes", "Köln"],
+		["Schwabing", "München"],
+	],
 	// R6: the FR instance. These are LIEUX-DITS under their communes (BAN `nom_ld`), not quartiers —
 	// see gazetteer-pipeline/lieudit-pairs.ts for why the French source differs from the US one.
 	fr: [

package/eval-harness/gauntlet/cases/regression.ts

@@ -1221,7 +1221,7 @@ export const REGRESSION_CASES: SeedCase[] = [
 			postcode: "TW6 2GA",
 		},
 		addedAt: "2026-08-01",
-		note: 'Sub-venue: TERMINAL is absent from Pub 28. Today: locality="Terminal", house_number=5, airport dropped.',
+		note: 'Sub-venue. IMPROVED 2026-08-02 by the WOF venue-structure vocabulary: unit="Terminal 5" is now correct (was locality="Terminal" + house_number=5, airport dropped). Residual: the airport reads as locality and Hounslow as street.',
 	},
 	{
 		id: "gb-subvenue-manchester-gate",
@@ -1237,7 +1237,7 @@ export const REGRESSION_CASES: SeedCase[] = [
 			postcode: "M90 1QX",
 		},
 		addedAt: "2026-08-01",
-		note: 'Sub-venue: TWO nested structural levels. Today: locality="Gate", house_number=12.',
+		note: 'Sub-venue, TWO nested structural levels. IMPROVED 2026-08-02: unit="Gate 12" is now correct (was locality="Gate" + house_number=12). Residual: only the inner level is captured.',
 	},
 	{
 		id: "gb-subvenue-st-thomas-wing",
@@ -1254,7 +1254,7 @@ export const REGRESSION_CASES: SeedCase[] = [
 			postcode: "SE1 7EH",
 		},
 		addedAt: "2026-08-01",
-		note: "Sub-venue with a TRAILING designator and a directional lead — the designator probe did not move this one.",
+		note: "Sub-venue with a TRAILING designator and a directional lead. UNMOVED by the venue-structure vocabulary: the proposer reads a designator that LEADS its identifier ('Wing B'), not one trailing a modifier ('West Wing'). A named residual.",
 	},
 	{
 		id: "us-subvenue-ohare-concourse",
@@ -1271,7 +1271,7 @@ export const REGRESSION_CASES: SeedCase[] = [
 			postcode: "60666",
 		},
 		addedAt: "2026-08-01",
-		note: "Sub-venue with a LETTER rather than a number; the designator probe split this into unit=Concourse + venue=B.",
+		note: "Sub-venue with a LETTER rather than a number. Still splits into unit='Concourse' + venue='B': the designator is recognized, the bare-letter identifier after it is not. The other named residual.",
 	},
 	{
 		id: "us-subvenue-googleplex-building",

package/gazetteer-pipeline/admin/ingest-wof.ts

@@ -13,6 +13,7 @@ import { readFile } from "node:fs/promises"
 import type { DatabaseSync } from "node:sqlite"
 
 import { isOfficialLanguage } from "@mailwoman/codex/country"
+import type { WOFFeature, WOFProperties } from "@mailwoman/core/resources/whosonfirst"
 import FastGlob from "fast-glob"
 import { asyncParallelIterator } from "spliterator"
 
@@ -60,8 +61,12 @@ interface ParsedFeature {
 }
 
 function parseFeature(text: string, placetypes: ReadonlySet<string>): ParsedFeature | null {
-	const feature = JSON.parse(text)
-	const props = feature.properties
+	// Typed against the schema `@mailwoman/core/resources/whosonfirst` already owns (`WOFFeature`/`WOFProperties`,
+	// admin.ts) rather than reading `any`. This file used to name all seventeen `wof:`/`geom:`/`edtf:`/`mz:` keys as
+	// bare strings, which meant a typo — `wof:superceded_by` — would have compiled, read `undefined`, and silently
+	// ingested every superseded record in the corpus. A type-only import: erased at build, zero runtime cost.
+	const feature = JSON.parse(text) as WOFFeature
+	const props: WOFProperties | undefined = feature.properties
 
 	if (!props) return null
 

package/gazetteer-pipeline/defaults.ts

@@ -23,6 +23,12 @@ export const DEFAULT_WOF_PRIORITY_COUNTRIES = [
 	"ES",
 	"FR",
 	"GB",
+	// Added 2026-08-02 after the granularity probe. `whosonfirst-data-admin-in` carries 189,026 sub-locality nodes —
+	// more than Germany's 67,162, the richest tier shipped before this — converting at 98.6% into 186,469 (child,
+	// parent) pairs, against Overture-IN's 74,920 nodes. That is 6× the shipped GB pair index, which took eight
+	// campaign rungs to assemble. IN moves OUT of DEFAULT_OVERTURE_COUNTRIES in the same change: a country served by
+	// both would double up its admin (the #267 warning).
+	"IN",
 	"IT",
 	"JP",
 	"KR",
@@ -72,7 +78,7 @@ export const DEFAULT_OVERTURE_COUNTRIES = [
 	"ID",
 	"IE",
 	"IL",
-	"IN",
+	// "IN" moved to DEFAULT_WOF_PRIORITY_COUNTRIES 2026-08-02 — see the note there.
 	"IQ",
 	"IR",
 	"IS",

package/gazetteer-pipeline/granularity-report.ts

@@ -0,0 +1,161 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Markdown renderer for the gazetteer depth scorecard — the committed artifact under
+ *   `docs/articles/evals/coverage/`, following the `fill-rates.md` precedent.
+ *
+ *   The report declares its own limits inline rather than deferring them to a design doc. These
+ *   numbers will outlive the conversation that produced them, and a reader six months out must be
+ *   told IN THE FILE that counts are not quality and that the locality rung is self-comparison for
+ *   the Overture-backfilled countries.
+ *
+ *   Pure: rows in, markdown out, `buildDate` injected by the caller so the output is deterministic
+ *   under test.
+ */
+
+import type { ComponentTag } from "@mailwoman/core/types"
+
+import { DEFAULT_WOF_PRIORITY_COUNTRIES } from "./defaults.ts"
+import { LADDER, type CountryGranularity, bottomsOutAt } from "./granularity.ts"
+
+/**
+ * Which build path supplied a country's rows — the single most important column in this report.
+ *
+ * A country whose rows are Overture- or GeoNames-sourced had NO WOF GeoJSON repo ingested, so its empty sub-locality
+ * rung says nothing whatsoever about WOF's depth there. Only a dozen of the 260 admin repos WOF publishes are in the
+ * recipe; the rest of the world arrives via Overture divisions (5 subtypes, none hood-level) or the GeoNames alias
+ * fold. Without this column a reader would take "233 countries bottom out at locality" as a finding about WOF rather
+ * than about the recipe.
+ *
+ * Derived from the ARTIFACT (synthetic id ranges on the locality rung), not from the recipe constants. A recipe-derived
+ * column silently goes wrong the moment a country is added to `DEFAULT_WOF_PRIORITY_COUNTRIES` and the gazetteer has
+ * not been rebuilt yet — it would claim `wof-repo` over rows that are still 100% Overture. Instead the recipe is used
+ * only as a CROSS-CHECK: a mismatch renders as `rebuild pending`, which is the honest description of that window.
+ */
+function sourceClass(country: CountryGranularity): string {
+	const locality = country.rungs.locality
+	const recipeSaysWOF = (DEFAULT_WOF_PRIORITY_COUNTRIES as readonly string[]).includes(country.country)
+
+	if (!locality?.nodes) return recipeSaysWOF ? "wof-repo (no rows)" : "(no rows)"
+
+	const synthetic = locality.overtureBackfilled + locality.geonamesBackfilled
+
+	const observed =
+		synthetic === 0 ? "wof-repo" : locality.geonamesBackfilled > locality.overtureBackfilled ? "geonames" : "overture"
+
+	// The recipe promises WOF rows the artifact does not yet carry: a rebuild has not run since the recipe changed.
+	if (recipeSaysWOF && observed !== "wof-repo") return `${observed} (rebuild pending)`
+
+	return observed
+}
+
+export interface GranularityReportMeta {
+	/**
+	 * Display path of the measured DB. Use the `$MAILWOMAN_DATA_ROOT`-relative form, never the resolved lab path.
+	 */
+	sourcePath: string
+	sourceMD5: string
+	/**
+	 * ISO timestamp, injected rather than read from the clock so the renderer stays deterministic.
+	 */
+	buildDate: string
+	floor: number
+}
+
+function pct(value: number): string {
+	return `${(value * 100).toFixed(1)}%`
+}
+
+/**
+ * One row's cell for a rung: node count, plus the Overture-backfilled share when any of it is backfilled.
+ */
+function rungCell(country: CountryGranularity, rung: ComponentTag): string {
+	const measurement = country.rungs[rung]
+
+	// Absent measurement = never measured. Distinct from a measured zero, which renders as "0".
+	if (!measurement) return "—"
+
+	if (!measurement.nodes) return "0"
+
+	const synthetic = measurement.overtureBackfilled + measurement.geonamesBackfilled
+
+	if (!synthetic) return measurement.nodes.toLocaleString()
+
+	// Label by whichever synthetic source dominates. A single `id >= OVERTURE_ID_BASE` test would call GeoNames rows
+	// Overture, which mislabelled every GeoNames-only country in the first run of this report.
+	const label = measurement.geonamesBackfilled > measurement.overtureBackfilled ? "gn" : "ovt"
+
+	return `${measurement.nodes.toLocaleString()} (${pct(synthetic / measurement.nodes)} ${label})`
+}
+
+/**
+ * Render the scorecard.
+ */
+export function renderGranularityReport(rows: CountryGranularity[], meta: GranularityReportMeta): string {
+	const header = LADDER.join(" | ")
+	const alignment = LADDER.map(() => "--:").join(" | ")
+
+	const body = rows.map((country) => {
+		const bottom = bottomsOutAt(country, meta.floor)
+		const cells = LADDER.map((rung) => rungCell(country, rung)).join(" | ")
+
+		return `| ${country.country} | ${sourceClass(country)} | ${bottom ?? "(none)"} | ${country.localityParents.toLocaleString()} | ${cells} |`
+	})
+
+	const reached = rows.filter((country) => bottomsOutAt(country, meta.floor) === "dependent_locality").length
+	const fromWOFRepo = rows.filter((country) => sourceClass(country) === "wof-repo").length
+
+	return `# Gazetteer depth scorecard
+
+Generated by \`mailwoman gazetteer granularity\`. Per-country measurement of where the admin
+gazetteer bottoms out — node counts per containment rung, and for the sub-locality rungs the
+parent-coverage share (the fraction of the country's locality-class nodes carrying at least one
+child projecting onto that rung).
+
+- **Source:** \`${meta.sourcePath}\` (md5 \`${meta.sourceMD5}\`)
+- **Built:** ${meta.buildDate}
+- **Parent-coverage floor:** ${pct(meta.floor)} — a sub-locality rung counts as reached only above this.
+- **Countries measured:** ${rows.length.toLocaleString()}
+- **Countries built from a cloned WOF repo:** ${fromWOFRepo.toLocaleString()} — the rest came from Overture divisions or the GeoNames alias fold.
+- **Countries reaching \`dependent_locality\`:** ${reached.toLocaleString()}
+
+## Read the \`source\` column first
+
+**This report measures the shipped artifact, not Who's on First.** WOF publishes 260 per-country
+admin repos; \`DEFAULT_WOF_PRIORITY_COUNTRIES\` names ${DEFAULT_WOF_PRIORITY_COUNTRIES.length} of
+them. Every other country's rows come from Overture divisions — whose \`OVERTURE_DIVISION_SUBTYPES\`
+is \`country\`/\`locality\`/\`region\`/\`county\`/\`localadmin\`, with no hood-level subtype — or from
+the GeoNames alias fold.
+
+So a \`source\` of \`overture\` or \`geonames\` with an empty \`dependent_locality\` rung is **not a
+finding about WOF's depth in that country**. It is a country we never asked. The sub-locality tier
+appears in exactly the countries whose repo is ingested, and that correspondence is the recipe, not a
+property of the data.
+
+The column is read off the **artifact** (synthetic id ranges), not the recipe. \`rebuild pending\`
+means the recipe now names a country the shipped gazetteer has not been rebuilt for — its rows are
+still Overture or GeoNames, and its rungs say nothing about WOF yet.
+
+## What this report does not tell you
+
+- **Counts are not quality.** A node count establishes where to look, never that the rows are good.
+- **There is no demand-side grounding below the locality line.** Nothing here proves a sub-locality
+  surface appears in real addresses. Overture's \`address_levels\` — the obvious instrument — bottoms
+  out at municipality in every country measured, so it cannot see this tier.
+- **The locality rung and above are partly self-comparison.** For the Overture-backfilled country
+  set those rows came from Overture, not WOF; the \`ovt\` share in each cell is how much.
+- **An empty rung is coverage, not fact.** Two independent filters produce zeroes here: the country
+  recipe above, and \`ADMIN_PLACETYPES\` in \`admin/ingest-wof.ts\`, which allowlists 9 of WOF's 34
+  placetypes — so even for a cloned country the build never asked for the other 25. Per the
+  **meaning-of-zero** rule a measured-and-empty rung renders as \`0\` and a never-measured rung as
+  \`—\`; they are not the same claim.
+
+## Ladder
+
+| country | source | bottoms out at | locality parents | ${header} |
+| --- | --- | --- | --: | ${alignment} |
+${body.join("\n")}
+`
+}