@mailwoman/resolver-wof-sqlite 10.8.0 → 11.1.0

package/README.md

@@ -37,7 +37,7 @@ for (const c of candidates) {
 
 Databases are `ATTACH`ed by a schema name **derived from the filename**, and queries route to them by matching that name against the requested placetype — `postalcode_us` serves `postalcode`. Two ways that used to fail without logging, and both now throw when you build the lookup:
 
-- The name does not route. `postcode-ca-overture.db` derives `postcode_ca_overture`, and the router tests `startsWith("postalcode_")` — **"postcode" is not "postalcode"**. It held 843,739 Canadian codes and answered every query with zero hits. A caller could not tell that result from "this country has no places".
+- The name doesn't route. `postcode-ca-overture.db` derives `postcode_ca_overture`, and the router tests `startsWith("postalcode_")` — **"postcode" isn't "postalcode"**. It held 843,739 Canadian codes and answered every query with zero hits. A caller couldn't tell that result from "this country has no places".
 - The database has `spr` but no `place_search`. It routes, then dies mid-`SELECT`.
 
 The predicate is the **table rather than the filename**: a database with an `spr` table is claiming to be a place database, and every lookup path here reaches the FTS index. A relation-table database like `postcode-locality-<cc>.db` has no `spr`, never makes that claim, and is exempt. That exemption keeps the documented default database list working.

package/lib/address/duplicate.ts

@@ -0,0 +1,133 @@
+// SPDX-FileCopyrightText: Teffen Ellis and Mailwoman contributors
+// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Commercial
+
+/**
+ *   Decides whether a lower-ranked address record repeats a higher-ranked record of
+ *   the same postcode, street and number, so the layer build may drop it.
+ */
+
+import { haversine } from "@mailwoman/spatial/geometries/point/distance"
+
+import { normalizeLocalityForKey } from "#street/normalize"
+
+/**
+ * The farthest apart two records' points may lie, in kilometers, while their localities disagree.
+ */
+export const ADDRESS_DUPLICATE_DISTANCE_KM = 2
+
+/**
+ * The most distinct addresses one source may place at a record's point
+ * before that point gives no distance evidence.
+ */
+export const ADDRESS_DUPLICATE_STACK_LIMIT = 5
+
+/**
+ * One record of a postcode key group, as the duplicate rule reads it.
+ */
+export interface DuplicateCandidate {
+	source: string
+	unit: string | null
+	number: string
+	streetRaw: string
+	localities: readonly string[]
+	lat: number
+	lon: number
+	/**
+	 * Distinct street-and-number pairs this record's source places at its point, rounded to 1e-5 degrees.
+	 */
+	stack: number
+}
+
+/**
+ * The parts of the duplicate rule a caller supplies for its country and sources.
+ */
+export interface AddressDuplicateRule {
+	/**
+	 * A source's rank, lowest first.
+	 * A record is dropped only for a record of a lower rank.
+	 */
+	rank(source: string): number
+	/**
+	 * The identifier a unit states, read against its record's number and street,
+	 * or `null` when it states none.
+	 */
+	unitIdentifier(unit: string | null, context: { number: string; street: string }): string | null
+}
+
+/**
+ * The keys of a record's stated localities, folded as the layer's key tables fold a locality name.
+ */
+export function addressLocalityKeys(localities: readonly string[]): Set<string> {
+	return new Set(localities.map((name) => normalizeLocalityForKey(name)).filter((key) => key.length > 0))
+}
+
+const localityKeys = (record: DuplicateCandidate) => addressLocalityKeys(record.localities)
+
+/**
+ * Whether `lower` repeats `higher`'s delivery point.
+ *
+ * The two must state the same unit by identifier, or both state none: a building row
+ * and a unit row are different delivery points.
+ * With the units agreeing, `lower` repeats `higher` unless both state localities that share
+ * no name and their points lie farther apart than {@linkcode ADDRESS_DUPLICATE_DISTANCE_KM}.
+ *
+ * A point stacked past {@linkcode ADDRESS_DUPLICATE_STACK_LIMIT} addresses gives no distance,
+ * so a pair touching one is a repeat whenever its units agree.
+ *
+ * Graded on #2978's held-out set of 397 adjudicated pairs from the 2026-11-08 US layer,
+ * this decides every pair as the EM-fitted Fellegi-Sunter model does: 263 true drops,
+ * 3 wrong drops and 3 missed repeats.
+ */
+export function isDuplicateAddress(
+	rule: AddressDuplicateRule,
+	higher: DuplicateCandidate,
+	lower: DuplicateCandidate
+): boolean {
+	return statesSameUnit(rule, higher, lower) && repeatsWithinPostcode(higher, lower)
+}
+
+/**
+ * Whether `lower` repeats `higher`'s delivery point when the two share a street and number
+ * but not a postcode group, because one of them states no postcode.
+ *
+ * Without a shared postcode, a locality agreement can join two towns' `100 Main Street`, so this
+ * needs the points within {@linkcode ADDRESS_DUPLICATE_DISTANCE_KM} whatever the localities say.
+ * A point stacked past {@linkcode ADDRESS_DUPLICATE_STACK_LIMIT} addresses gives
+ * no distance, so a pair touching one isn't a repeat.
+ */
+export function isNearbyDuplicateAddress(
+	rule: AddressDuplicateRule,
+	higher: DuplicateCandidate,
+	lower: DuplicateCandidate
+): boolean {
+	if (!statesSameUnit(rule, higher, lower)) return false
+
+	if (higher.stack > ADDRESS_DUPLICATE_STACK_LIMIT || lower.stack > ADDRESS_DUPLICATE_STACK_LIMIT) return false
+
+	return haversine(higher.lat, higher.lon, lower.lat, lower.lon) <= ADDRESS_DUPLICATE_DISTANCE_KM
+}
+
+/**
+ * Whether two records state the same unit by identifier, or both state none.
+ */
+function statesSameUnit(rule: AddressDuplicateRule, higher: DuplicateCandidate, lower: DuplicateCandidate): boolean {
+	const unit = (record: DuplicateCandidate) =>
+		rule.unitIdentifier(record.unit, { number: record.number, street: record.streetRaw })
+
+	return unit(higher) === unit(lower)
+}
+
+/**
+ * Whether two records of one postcode key group are one delivery point, once their units agree.
+ */
+function repeatsWithinPostcode(higher: DuplicateCandidate, lower: DuplicateCandidate): boolean {
+	const a = localityKeys(higher)
+	const b = localityKeys(lower)
+	const disjoint = a.size > 0 && b.size > 0 && ![...a].some((name) => b.has(name))
+
+	if (!disjoint) return true
+
+	if (higher.stack > ADDRESS_DUPLICATE_STACK_LIMIT || lower.stack > ADDRESS_DUPLICATE_STACK_LIMIT) return true
+
+	return haversine(higher.lat, higher.lon, lower.lat, lower.lon) <= ADDRESS_DUPLICATE_DISTANCE_KM
+}

package/lib/address/keys/attribution.ts

@@ -0,0 +1,119 @@
+// SPDX-FileCopyrightText: Teffen Ellis and Mailwoman contributors
+// SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Commercial
+
+/**
+ *   Writes the `ATTRIBUTION.json` that sits beside a built address-layer region's files.
+ */
+
+import { writeLocalJSONFile } from "@mailwoman/core/fs/writers"
+import { DatabaseClient } from "@mailwoman/sqlite/client"
+import { sql } from "kysely"
+import { dirname, resolvePathBuilder, type PathBuilder, type PathBuilderLike } from "path-ts"
+
+import { addressLayerFiles } from "#address/keys/layout"
+import type { AddressDetailDatabase, AddressLookupDatabase } from "#address/keys/schema"
+
+/**
+ * One source and authority of a region's rows, with the terms the source records.
+ */
+export interface AddressLayerAttribution {
+	/**
+	 * The wire source id, such as `usgov-nad` or `openaddresses:us/wi/statewide`.
+	 */
+	source: string
+	/**
+	 * The authority a row states, such as NAD's `AddAuth`, or `null` when its rows state none.
+	 */
+	authority: string | null
+	records: number
+	/**
+	 * The license expression `lookup.db`'s `source` table holds, or `null`
+	 * when the source's terms are unread.
+	 */
+	license: string | null
+	/**
+	 * The terms as the source's definition records them, such as an OpenAddresses run's
+	 * `license` object, or `null` when the caller has no definition for the source.
+	 */
+	recorded: unknown
+	/**
+	 * The distinct values this source's rows state as their authority, when those values
+	 * identify records rather than authorities (see {@linkcode AUTHORITY_RECORD_ID_SHARE})
+	 * and so are left out of this entry.
+	 * Absent when the source's authorities are kept.
+	 */
+	omittedAuthorities?: number
+}
+
+/**
+ * The share of a source's rows that may each state an authority value no other row shares
+ * before the source's authority values are read as record identifiers.
+ *
+ * An authority is a body that maintains a whole area's addresses, so a source's
+ * rows share a few authority values.
+ * In NAD release 24, every state but one holds at most one distinct `AddAuth` value per 5,000 rows.
+ *
+ * Idaho holds 78,956 distinct values across 79,656 rows: its provider wrote each record's id into the field.
+ */
+export const AUTHORITY_RECORD_ID_SHARE = 0.5
+
+/**
+ * Writes `ATTRIBUTION.json` beside a built region's `lookup.db`, one entry per source
+ * and authority, largest first.
+ *
+ * Row counts come from `detail.db`, and each license from `lookup.db`'s `source` table.
+ *
+ * @param recorded The terms each source's definition records, keyed by source id.
+ */
+export async function writeAddressLayerAttribution(
+	root: PathBuilderLike,
+	region: string,
+	version: string,
+	recorded: ReadonlyMap<string, unknown> = new Map()
+): Promise<{ path: PathBuilder; entries: AddressLayerAttribution[] }> {
+	const files = addressLayerFiles(root, region, version)
+
+	using lookup = new DatabaseClient<AddressLookupDatabase>(files.lookup, { readOnly: true })
+	using detail = new DatabaseClient<AddressDetailDatabase>(files.detail, { readOnly: true })
+
+	const sources = new Map(
+		(await lookup.selectFrom("source").select(["source_id", "name", "license"]).execute()).map((row) => [
+			row.source_id,
+			row,
+		])
+	)
+
+	const tallies = await detail
+		.selectFrom("address")
+		.select(["source_id", "authority", sql<number>`count(*)`.as("records")])
+		.groupBy(["source_id", "authority"])
+		.execute()
+
+	const bySource = Map.groupBy(tallies, (row) => row.source_id)
+
+	const entries = [...bySource]
+		.flatMap(([sourceID, rows]): AddressLayerAttribution[] => {
+			const source = sources.get(sourceID)
+
+			if (!source) throw new Error(`${files.detail} holds rows of source ${sourceID}, which lookup.db lacks`)
+
+			const base = { source: source.name, license: source.license, recorded: recorded.get(source.name) ?? null }
+			const records = rows.reduce((sum, row) => sum + Number(row.records), 0)
+			const authorities = rows.filter((row) => row.authority != null).length
+			const singletons = rows.filter((row) => row.authority != null && Number(row.records) === 1).length
+
+			// A source most of whose rows each state an authority value no other row shares is stating record ids there.
+			if (authorities > 1 && singletons > records * AUTHORITY_RECORD_ID_SHARE) {
+				return [{ ...base, authority: null, records, omittedAuthorities: authorities }]
+			}
+
+			return rows.map((row) => ({ ...base, authority: row.authority, records: Number(row.records) }))
+		})
+		.toSorted((left, right) => right.records - left.records)
+
+	const path = resolvePathBuilder(dirname(files.lookup), "ATTRIBUTION.json")
+
+	await writeLocalJSONFile(entries, path)
+
+	return { path, entries }
+}