@mailwoman/resolver-wof-sqlite 9.4.0 → 10.1.0

package/lib/candidate/name-roles.ts

@@ -9,24 +9,22 @@ import { isOfficialLanguage } from "@mailwoman/codex/country"
 import type { DatabaseClient } from "@mailwoman/sqlite/client"
 import { tableExists } from "@mailwoman/sqlite/introspection"
 
-import type { CandidateDatabase } from "#candidate-schema"
 import { isOwnNameVariant } from "#candidate/own-name"
 import type { PlaceAttrs } from "#candidate/place-attrs"
+import type { CandidateDatabase } from "#candidate/schema"
 import type { WOFDatabase } from "#schema"
 import { normalizeLocalityForKey } from "#street/normalize"
 
 /**
- * Key-count threshold for the gloss anomaly detector (#1730) — the sweep's own boundary: 4,000 places carried >= 50
- * keys, and a legitimate famous place at that count (New York, 176 keys) is separated by the PROMINENCE check, never by
- * this number alone.
+ * Key-count threshold for the gloss anomaly detector.
+ *
+ * A legitimate famous place is separated by the prominence check rather than this number by itself.
  */
 export const GLOSS_KEY_THRESHOLD = 50
 
 /**
- * Placetypes the gloss detector never flags. A country or region legitimately carries a name in every language — that
- * is what an exonym set IS — so key volume discriminates nothing there. The detector's population is the non-admin
- * tail, where a place named by a common noun ("Poisson", "Sunday") accumulating 200+ translations is a
- * machine-translated gloss set, not fame.
+ * Placetypes the gloss detector never flags: a country or region legitimately has a
+ * name in every language, so key volume provides no signal there.
  */
 export const GLOSS_EXCLUDED_PLACETYPES: ReadonlySet<string> = new Set([
 	"country",
@@ -40,21 +38,11 @@ export const GLOSS_EXCLUDED_PLACETYPES: ReadonlySet<string> = new Set([
 ])
 
 /**
- * Pass 3c — the #1730 name-role prototype: two independent detectors over the staged rows, WRITE-ONLY in this
- * generation (no ranking consumer; the rank penalty is its own D-rule-conditional step with the `gloss_key` board as
- * regression check). Both stamp `is_primary = 0` rows only — a place's canonical name and the `place_abbr` region
- * abbreviations are never a gloss or a variant.
- *
- * - `gloss` is ANOMALY-based, and stamps only the certain core: key volume at/over the threshold + a non-admin placetype
- *
- *   - NO measured prominence (population absent AND importance unmeasured). Provenance cannot separate a gloss from an
- *     exonym — WOF imported both as `x_preferred` — and prominence is what rescues New York/Paris.
- * - `abbr` is PROVENANCE-based — the #936 signal: a WOF `variant` name in one of the country's official languages (or
- *   English), measured there at a 13× key-collision rate. A source without a `names` table (fixture-scale admin DBs)
- *   skips this detector loudly.
+ * Stamp `name_role` on non-primary rows: `abbr` by provenance, `variant` by
+ * own-name verdict, then `gloss` by key-volume anomaly.
  *
- * Returns the stamp counts plus the census the prototype exists to report: how much of the ≥-threshold key tail carries
- * any role.
+ * @returns The stamp counts plus the census the prototype exists to report:
+ * how much of the ≥-threshold key tail has any role.
  */
 export function stampNameRoles(ctx: {
 	src: DatabaseClient<WOFDatabase>
@@ -85,7 +73,7 @@ export function stampNameRoles(ctx: {
 
 		if (GLOSS_EXCLUDED_PLACETYPES.has(ptNameByID.get(a.ptid) ?? "")) continue
 
-		if (a.pop > 0 || a.imp != null) continue
+		if ((a.pop ?? 0) > 0 || a.imp != null) continue
 		glossSids.push(sid)
 	}
 
@@ -101,9 +89,8 @@ export function stampNameRoles(ctx: {
 	if (hasSourceNames) {
 		out.exec("BEGIN")
 
-		// Two provenance routes into the same stamp: WOF's abbreviation/short name KINDS arrive in the
-		// LANGUAGE column ('abbr'/'short' — 280 rows, measured 2026-08-18, Toledo's 'TO' among them) and
-		// qualify by kind alone; everything else qualifies as a variant in an official language.
+		// WOF's abbreviation/short name kinds qualify by kind only.
+		// Everything else qualifies as a variant in an official language.
 		for (const r of src
 			.prepare("SELECT id, name, language FROM names WHERE privateuse = 'variant' OR language IN ('abbr', 'short')")
 			.iterate()) {
@@ -129,8 +116,7 @@ export function stampNameRoles(ctx: {
 		progress("roles", "source carries no `names` table — abbr detector skipped (gloss still runs)")
 	}
 
-	// Stamp order is precedence: the provenance-based abbr first, then the own-name variant verdict,
-	// then gloss fills what neither claimed.
+	// Stamp order is precedence: abbr first, then the own-name variant verdict, then gloss fills what neither claimed.
 	const roleAbbr = Number(
 		out
 			.prepare(
@@ -141,12 +127,8 @@ export function stampNameRoles(ctx: {
 			.run().changes
 	)
 
-	// --- variant detector (#1882): the alias surface is the holder's OWN primary name in another
-	// orthography — romanization, spacing/diacritic variant, or abbreviation expansion. The verdict
-	// is per (alias key, primary key) pair, so it runs in JS over the still-unstamped alias rows;
-	// an uncovered script answers no-verdict and stamps nothing (own-name.ts owns the predicate and
-	// its measured threshold). Runs BEFORE gloss on purpose: a surface that IS the place's own name
-	// is not a translation, whatever the key volume says.
+	// Runs before gloss so the place's own name remains its source name.
+	// An uncovered script answers no-verdict.
 	out.exec(
 		"CREATE TEMP TABLE variant_key (spr_id INTEGER NOT NULL, name_key TEXT NOT NULL, PRIMARY KEY (spr_id, name_key)) WITHOUT ROWID"
 	)

package/lib/candidate/own-name.ts

@@ -3,40 +3,36 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   The OWN-NAME VARIANT predicate (#1882): is an alias surface the holder's own primary name in
- *   another orthography — a romanization (`Брэст` → `brest`), a spacing/diacritic variant
- *   (`George Town` → `georgetown`), an abbreviation expansion (`St. George's` → `Saint George's`)
- *   — rather than a different name that merely shares the folded key?
+ *   The own-name variant predicate: is an alias surface the holder's own primary name in another
+ *   orthography, whether a romanization (`Брэст` → `brest`), a spacing or diacritic variant
+ *   (`George Town` → `georgetown`), or an abbreviation expansion (`St. George's` → `Saint George's`)?
+ *   The alternative is a different name that merely shares the folded key.
  *
- *   The comparator is `levenshteinSimilarity`, NOT Jaro-Winkler: JW's common-prefix bonus scores
- *   the required NEGATIVE case (`chanchun` vs `cancun`, 0.925) above real positives
- *   (`saint george s` vs `st georges` expanded, 0.914), so no JW threshold separates them.
- *   Measured on the #1882 census contests, edit similarity separates every case with margin:
+ *   The comparator is `levenshteinSimilarity`. Jaro-Winkler's common-prefix bonus scores the required
+ *   negative case (`chanchun` against `cancun`, 0.925) above real positives (`saint george s` against
+ *   `st georges` expanded, 0.914), so no JW threshold separates them. Edit similarity separates every
+ *   measured case with margin.
  *
- *   - IN: `brest`/`brest` 1.0 · `george town`/`georgetown` 0.909 · `saint george s`/`saint georges`
- *       0.929 · `adamovka`/`adamowka` 0.875
- *   - OUT: `lievin`/`levin` 0.833 (Liévin FR vs Levin NZ — different places with near-identical
- *       names; the panel's `41 Weraroa Road, Levin` row needs this side) · `chanchun`/`cancun` 0.75 ·
- *       `augsburg`/`augusta` 0.375 · `west bay`/`west end` 0.625 · `derry`/`londonderry` 0.455
- *       (Derry/Londonderry is a DUAL NAME, not a variant — its own follow-up on #1882)
- *
- *   An unhandled script (Arabic, Hebrew, CJK — the romanizer covers Cyrillic only) answers NULL,
- *   never "different name": absence of a verdict must not stamp anything (the meaning-of-zero rule).
- *   The #1882 census counted 396 demoted Arabic/Hebrew-primary aliases left unclassified by this.
+ *   An unhandled script (Arabic, Hebrew, CJK, since the romanizer covers Cyrillic only) answers NULL,
+ *   never "different name". Absence of a verdict must not stamp anything.
  */
 
 import { levenshteinSimilarity } from "@mailwoman/match/comparators"
 
 /**
- * Edit-similarity floor for the variant verdict. The measured band: nearest admitted pair 0.875
- * (`adamovka`/`adamowka`), nearest refused pair 0.833 (`lievin`/`levin`).
+ * Edit-similarity floor for the variant verdict.
+ *
+ * The measured band: nearest admitted pair 0.875 (`adamovka`/`adamowka`),
+ * nearest refused pair 0.833 (`lievin`/`levin`).
  */
 export const VARIANT_SIMILARITY_MIN = 0.85
 
 /**
- * BGN/PCGN-flavored Cyrillic romanization, folded to the name-key alphabet. Digraph outputs (zh, kh, ts, ch, sh, shch,
- * yu, ya) match the dominant transliteration conventions the gazetteer's Latin aliases actually use; the `w`/`v` and
- * `kh`/`h` style variance between systems is what the edit-similarity threshold absorbs.
+ * BGN/pcgn-flavored Cyrillic romanization, folded to the name-key alphabet.
+ *
+ * Digraph outputs (zh, kh, ts, ch, sh, shch, yu, ya) match the dominant transliteration
+ * conventions the gazetteer's Latin aliases actually use.
+ * The `w`/`v` and `kh`/`h` style variance between systems is what the edit-similarity threshold absorbs.
  */
 const CYRILLIC_TO_LATIN: Record<string, string> = {
 	а: "a",
@@ -80,8 +76,10 @@ const CYRILLIC_TO_LATIN: Record<string, string> = {
 }
 
 /**
- * Leading-word abbreviations expanded BEFORE comparison, so `st georges` meets `saint george s` inside the edit
- * threshold. Whole-word only — `st` inside `stanley` never expands.
+ * Leading-word abbreviations expanded before comparison, so `st georges` meets
+ * `saint george s` inside the edit threshold.
+ *
+ * Whole-word only, so `st` inside `stanley` never expands.
  */
 const NAME_ABBREVIATIONS: ReadonlyArray<[RegExp, string]> = [
 	[/\bst\b/g, "saint"],
@@ -91,7 +89,7 @@ const NAME_ABBREVIATIONS: ReadonlyArray<[RegExp, string]> = [
 ]
 
 /**
- * Expand the whole-word abbreviations above.
+ * Expand the whole-word abbreviations above in a name key.
  */
 export function expandNameAbbreviations(key: string): string {
 	let out = key
@@ -104,8 +102,10 @@ export function expandNameAbbreviations(key: string): string {
 }
 
 /**
- * Romanize a folded name key to the a–z0–9/space alphabet. `null` when characters outside the covered scripts remain —
- * an unhandled script is NO VERDICT, not a mismatch.
+ * Romanize a folded name key to the a–z0–9/space alphabet.
+ *
+ * `null` when characters outside the covered scripts remain.
+ * An unhandled script is no verdict.
  */
 export function romanizeNameKey(key: string): string | null {
 	let out = ""
@@ -124,10 +124,13 @@ export function romanizeNameKey(key: string): string | null {
 }
 
 /**
- * Edit similarity between a holder's primary name key and one of its alias keys, both romanized and
- * abbreviation-expanded. `null` when either side's script is uncovered.
+ * Edit similarity between a holder's primary name key and one of its alias keys,
+ * both romanized and abbreviation-expanded.
+ *
+ * `null` when either side's script is uncovered.
  */
-// repo-health-ignore export-name-affix -- romanizes and expands abbreviations first; `nameSimilarity` compares as given.
+// repo-health-ignore export-name-affix -- romanizes and expands abbreviations first;
+// `nameSimilarity` compares as given.
 export function ownNameSimilarity(primaryKey: string, aliasKey: string): number | null {
 	const primary = romanizeNameKey(primaryKey)
 	const alias = romanizeNameKey(aliasKey)

package/lib/candidate/place-attrs.ts

@@ -4,12 +4,11 @@
  * @author Teffen Ellis, et al.
  * @file The per-place record every candidate-staging pass writes its rows from.
  *
- *   Pass 1 reduces each current `spr` row to one {@link PlaceAttrs}; every later pass (alias bags,
+ *   Pass 1 reduces each current `spr` row to one {@link PlaceAttrs}. Every later pass (alias bags,
  *   region abbreviations, country display names, the currency backfill, the extract folds) discovers
- *   ADDITIONAL name keys for a place already in that map and stages a row against the same record.
- *   That is what keeps each candidate row denormalized without re-reading the source, and it is why
- *   a pass needs exactly four things to stage: the key it found, the place, the id the row hangs on,
- *   and whether the key is the place's canonical name.
+ *   additional name keys for a place already in that map and stages a row against the same record.
+ *   That keeps each candidate row denormalized without re-reading the source. A pass therefore needs
+ *   exactly four values: the key, place, row ID, plus a flag for the place's canonical name.
  */
 
 export interface PlaceAttrs {
@@ -23,22 +22,30 @@ export interface PlaceAttrs {
 	mnLon: number
 	mxLat: number
 	mxLon: number
-	pop: number
+	/**
+	 * The place's recorded population, or null when the gazetteer never measured one.
+	 *
+	 * NULL rather than zero.
+	 * `place_population` holds no zero, so an absent row is the only way a place has no number.
+	 * A zero written for those would be an unmeasured count.
+	 */
+	pop: number | null
 	neg: number
 	pkey: string
 	/**
-	 * The place's toponym-fame score, or null when the score source has no measurement for it (#28). A property of the
-	 * PLACE, so it rides {@link StageRow} onto the alias and abbrev rows too — that is how a bare `Moscow` reaches
-	 * Москва's score through the alias row that carries the key.
+	 * The place's toponym-fame score, or null when the score source has no measurement for it.
+	 *
+	 * A property of the place, so it appears on {@link StageRow} and on alias and abbrev rows too.
+	 * That is how a bare `Moscow` reaches Москва's score through the alias row that includes the key.
 	 */
 	imp: number | null
 }
 
 /**
- * Stage one candidate row: a normalized name key, the place it belongs to, the source id the row hangs on, and whether
- * the key is that place's canonical name (`is_primary`).
+ * Stages one candidate row with a normalized name key, place, source ID,
+ * plus a canonical-name flag (`is_primary`).
  *
- * `sid` is passed separately rather than read off the place because a extract fold and the alias pass stage rows for
- * ids the admin `attrs` map never held.
+ * `sid` is passed separately rather than read off the place because an extract fold
+ * and the alias pass stage rows for ids the admin `attrs` map never held.
  */
 export type StageRow = (k: string, a: PlaceAttrs, sid: number, isPrimary: number) => void

package/lib/candidate/schema.ts

@@ -0,0 +1,211 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ */
+
+import { sql, type Kysely } from "kysely"
+
+import type { CandidateAncestorsDatabase } from "#candidate/ancestors/schema"
+import type { CapitalTable } from "#capital-schema"
+import type { NameKey } from "#street/normalize"
+
+/**
+ * One candidate row.
+ *
+ * `name_key`, four small integer keys, `neg_rank`, and `spr_id` form the clustered primary key.
+ * The remaining fields are denormalized so a resolve needs one probe.
+ */
+export interface CandidateTable {
+	/**
+	 * The shared {@link normalizeLocalityForKey} of the name/alias — the probe key.
+	 */
+	name_key: NameKey
+	/**
+	 * Small int from {@link CountryCodeTable} (shrinks the clustered key).
+	 */
+	country_id: number
+	/**
+	 * The place's region-tier ancestor id, or 0.
+	 */
+	region_id: number
+	/**
+	 * Small int from {@link PlacetypeCodeTable}.
+	 */
+	placetype_id: number
+	/**
+	 * `-log10(population + 1)`.
+	 *
+	 * The sort uses ascending order to put the highest population first.
+	 * Postcodes use `0`.
+	 */
+	neg_rank: number
+	/**
+	 * WOF id of the place this row resolves to.
+	 */
+	spr_id: number
+	name: string | null
+	latitude: number | null
+	longitude: number | null
+	min_lat: number | null
+	min_lon: number | null
+	max_lat: number | null
+	max_lon: number | null
+	population: number | null
+	/**
+	 * 1 when the row is the place's canonical name (vs an alias/abbrev).
+	 */
+	is_primary: number | null
+	/**
+	 * Blended place importance in [0, 1], NULL when unmeasured (never zero) and copied
+	 * verbatim from the pre-split score source rather than the split `encyclopedic` channel.
+	 */
+	importance: number | null
+	/**
+	 * The name's detected role on this row (`'abbr'`, `'gloss'` or `'variant'`),
+	 * or NULL when no role was detected.
+	 * The column is write-only in this build generation.
+	 */
+	name_role: string | null
+}
+
+/**
+ * `(id → ISO country code)` dictionary.
+ */
+export interface CountryCodeTable {
+	id: number
+	code: string
+}
+
+/**
+ * `(id → placetype)` dictionary.
+ */
+export interface PlacetypeCodeTable {
+	id: number
+	placetype: string
+}
+
+/**
+ * The candidate database schema, extending the ancestors sidecar so one typed
+ * client covers every table in the artifact.
+ */
+export interface CandidateDatabase extends CandidateAncestorsDatabase {
+	/**
+	 * The clustered `without rowid` lookup table the reader probes.
+	 */
+	candidate: CandidateTable
+	/**
+	 * Transient staging table (same columns); dropped once `candidate` is materialized.
+	 */
+	cand_stage: CandidateTable
+	country_codes: CountryCodeTable
+	placetype_codes: PlacetypeCodeTable
+	/**
+	 * The capital-status reference stored in the artifact.
+	 * See capital-schema.ts.
+	 */
+	capital: CapitalTable
+}
+
+/**
+ * The `candidate`/`cand_stage` columns in clustered-key order, from which the
+ * materialization derives its column list.
+ * Keep in sync with {@link CandidateTable}.
+ */
+export const CANDIDATE_COLUMNS = [
+	"name_key",
+	"country_id",
+	"region_id",
+	"placetype_id",
+	"neg_rank",
+	"spr_id",
+	"name",
+	"latitude",
+	"longitude",
+	"min_lat",
+	"min_lon",
+	"max_lat",
+	"max_lon",
+	"population",
+	"is_primary",
+	// Appended, never inserted mid-list: the positional `insert into cand_stage values (…)` binds by position.
+	"importance",
+	"name_role",
+] as const
+
+/**
+ * Create the code dictionaries and the transient staging table; `cand_stage` mirrors
+ * {@link CandidateTable} with every column nullable because the loader fills them positionally.
+ */
+export async function createCandidateStagingTables(db: Kysely<CandidateDatabase>): Promise<void> {
+	await db.schema
+		.createTable("country_codes")
+		.addColumn("id", "integer", (c) => c.primaryKey())
+		.addColumn("code", "text", (c) => c.unique())
+		.execute()
+
+	await db.schema
+		.createTable("placetype_codes")
+		.addColumn("id", "integer", (c) => c.primaryKey())
+		.addColumn("placetype", "text", (c) => c.unique())
+		.execute()
+
+	await db.schema
+		.createTable("cand_stage")
+		.addColumn("name_key", "text")
+		.addColumn("country_id", "integer")
+		.addColumn("region_id", "integer")
+		.addColumn("placetype_id", "integer")
+		.addColumn("neg_rank", "real")
+		.addColumn("spr_id", "integer")
+		.addColumn("name", "text")
+		.addColumn("latitude", "real")
+		.addColumn("longitude", "real")
+		.addColumn("min_lat", "real")
+		.addColumn("min_lon", "real")
+		.addColumn("max_lat", "real")
+		.addColumn("max_lon", "real")
+		.addColumn("population", "integer")
+		.addColumn("is_primary", "integer")
+		.addColumn("importance", "real")
+		.addColumn("name_role", "text")
+		.execute()
+}
+
+/**
+ * Create the clustered `without rowid` lookup table.
+ *
+ * Its first six columns form the primary key, ranked by population through `neg_rank`.
+ */
+export async function createCandidateTable(db: Kysely<CandidateDatabase>): Promise<void> {
+	await db.schema
+		.createTable("candidate")
+		.addColumn("name_key", "text", (c) => c.notNull())
+		.addColumn("country_id", "integer", (c) => c.notNull())
+		.addColumn("region_id", "integer", (c) => c.notNull())
+		.addColumn("placetype_id", "integer", (c) => c.notNull())
+		.addColumn("neg_rank", "real", (c) => c.notNull())
+		.addColumn("spr_id", "integer", (c) => c.notNull())
+		.addColumn("name", "text")
+		.addColumn("latitude", "real")
+		.addColumn("longitude", "real")
+		.addColumn("min_lat", "real")
+		.addColumn("min_lon", "real")
+		.addColumn("max_lat", "real")
+		.addColumn("max_lon", "real")
+		.addColumn("population", "integer")
+		.addColumn("is_primary", "integer")
+		.addColumn("importance", "real")
+		.addColumn("name_role", "text")
+		.addPrimaryKeyConstraint("candidate_pk", [
+			"name_key",
+			"country_id",
+			"region_id",
+			"placetype_id",
+			"neg_rank",
+			"spr_id",
+		])
+		// `without rowid` has no first-class builder. The raw modifier is the idiomatic fallback.
+		.modifyEnd(sql`without rowid`)
+		.execute()
+}