@mailwoman/corpus 9.4.0 → 10.0.0

package/lib/adapters/usgov-samhsa-treatment-locator/adapter.ts

@@ -1,193 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `usgov-samhsa-treatment-locator`: SAMHSA Behavioral Health Treatment Services Locator CSV
- *   consumer.
- *
- *   SAMHSA's Treatment Locator (`findtreatment.gov`) is the federal directory of substance-use and
- *   mental-health treatment facilities. The published CSV carries the facility name, an optional
- *   secondary name (typically the organizational parent), and the postal address quad split into
- *   primary + secondary street lines. Phase 1.6 §1.2 (#22) selects this source for the same reason
- *   it selects HRSA: facility names are hand-typed venue strings and the addresses pass through
- *   enough human + system hands to accumulate the suite-designator + sub-tenant chaos ("Suite C,
- *   behind main building") that pure gazetteer data does not.
- *
- *   SAMHSA's two-line address shape is the key adapter-specific concern. `street1` typically carries
- *   the canonical postal address (`"123 Main St"`); `street2` carries the suite / unit / "second
- *   floor" surface form. The adapter joins them with `", "` into a single `street` component (Phase
- *   1 keeps `unit` as a deferred slot since the OpenCage template doesn't have a clean rendering
- *   for it). Operators wanting a different join policy can subclass the factory.
- *
- *   Column names below match the canonical SAMHSA Behavioral Health Treatment Services Locator CSV
- *   export header. Operators substituting a closely-related extract should rename columns to match;
- *   the README has the mapping cheatsheet.
- *
- *   License: stamped `"Public Domain"` per the SAMHSA Open Data Foundry's federal-government
- *   distribution terms.
- */
-
-import { reconcileComponents } from "@mailwoman/formatter"
-import { CSVSpliterator } from "spliterator"
-
-import { composeRaw, splitStreetLine, stableSourceID } from "#adapters/utils"
-import { lookupStateAbbreviation } from "#codex/us-fips-state"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator"
-/**
- * License carried by this source (Public Domain), attached to each row so downstream consumers inherit the terms rather
- * than having to look them up.
- */
-export const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain"
-
-/**
- * Subset of SAMHSA Treatment Locator CSV columns consulted by the adapter. Column names match the canonical SAMHSA Open
- * Data Foundry export header. `name1` is the venue; `name2` is optional and folded into the venue when present.
- */
-interface SamhsaSiteRow {
-	name1: string
-	name2?: string
-	street1: string
-	street2?: string
-	city: string
-	state: string
-	zip: string
-	/**
-	 * Optional. Falls back to `stableSourceID` derived from components when missing.
-	 */
-	frid?: string
-}
-
-/**
- * Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
- * building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
- * the `unit` component — see the file-level comment.
- */
-function joinTwoLineStreet(street1: string, street2: string | undefined): string {
-	const s1 = street1.trim()
-	const s2 = (street2 ?? "").trim()
-
-	if (!s1 && !s2) return ""
-
-	if (!s2) return s1
-
-	if (!s1) return s2
-
-	return `${s1}, ${s2}`
-}
-
-/**
- * Combine `name1` + optional `name2` into a single venue surface form. SAMHSA conventions:
- *
- * - `name1` is the program / clinic name ("Mountain Plains Counseling Services").
- * - `name2` is the parent organization ("Catholic Charities of Wyoming"), if any.
- *
- * Both render together as `"<name1> - <name2>"` when both are present — geocoder users typically type either form, so
- * the model benefits from the joined surface.
- */
-function composeVenue(name1: string, name2: string | undefined): string {
-	const n1 = name1.trim()
-	const n2 = (name2 ?? "").trim()
-
-	if (!n1 && !n2) return ""
-
-	if (!n2) return n1
-
-	if (!n1) return n2
-
-	return `${n1} - ${n2}`
-}
-
-export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
-	return {
-		id: USGOV_SAMHSA_ADAPTER_ID,
-		defaultLicense: USGOV_SAMHSA_DEFAULT_LICENSE,
-		description:
-			"SAMHSA Behavioral Health Treatment Services Locator (public-domain). Adversarial source: venue + two-line address co-occurrence, hand-entered.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			if (opts.country && opts.country !== "US") {
-				throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
-			}
-
-			const rows = CSVSpliterator.fromAsync(opts.inputPath, {
-				mode: "object",
-				normalizeKeys: false,
-				enableQuoteHandling: true,
-			})
-
-			let emitted = 0
-
-			for await (const record of rows as AsyncIterable<SamhsaSiteRow>) {
-				if (opts.signal?.aborted) break
-
-				if (opts.limit !== undefined && emitted >= opts.limit) break
-
-				const venue = composeVenue(record.name1 ?? "", record.name2)
-				const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
-				const split = splitStreetLine(street)
-				const city = (record.city ?? "").trim()
-				const stateAbbr = (record.state ?? "").trim()
-				const postcode = (record.zip ?? "").trim()
-
-				if (!venue || !split || !city || !postcode) continue
-				const state = lookupStateAbbreviation(stateAbbr)
-
-				if (!state) continue
-
-				// venue first — same kryptonite-defending insertion order as HRSA.
-				const components: CanonicalRow["components"] = {
-					venue,
-					...(split.house_number ? { house_number: split.house_number } : {}),
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode,
-				}
-
-				const raw = composeRaw({
-					venue,
-					houseNumber: split.house_number,
-					street: split.street,
-					locality: city,
-					region: state.abbreviation,
-					postcode,
-				})
-
-				if (!raw) continue
-
-				const aligned = reconcileComponents(components, raw)
-
-				if (!Object.keys(aligned).length) continue
-
-				const frID = (record.frid ?? "").trim()
-
-				const sourceID = frID ? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}` : stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
-
-				yield {
-					raw,
-					components: aligned,
-					country: "US",
-					locale: "en-US",
-					source: USGOV_SAMHSA_ADAPTER_ID,
-					source_id: sourceID,
-					corpus_version: "",
-					license: USGOV_SAMHSA_DEFAULT_LICENSE,
-				}
-
-				emitted++
-			}
-		},
-	}
-}
-
-/**
- * The configured adapter instance registered with the corpus builder.
- */
-export const usgovSamhsaTreatmentLocatorAdapter = createUsgovSamhsaTreatmentLocatorAdapter()

package/lib/adapters/wof-admin-json/adapter.ts

@@ -1,231 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   `wof-admin`: Who's On First admin GeoJSON-bundle adapter.
- *
- *   **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at
- *   `packages/corpus/lib/adapters/wof-admin/`, removed in this same change) was replaced by this
- *   one because the SQLite distribution path was unworkable for the real corpus build:
- *
- *   1. `dist.whosonfirst.org/sqlite/` is dead (NXDOMAIN); the Geocode-Earth mirror is the only one.
- *   2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated
- *        as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows.
- *   3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a
- *        separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases
- *        (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a
- *        patched `is_current` predicate.
- *
- *   Input: a directory containing one or more cloned `whosonfirst-data-admin-<cc>` GitHub repos. Each
- *   repo has `data/XXX/YYY/ZZZ/<wof-id>.geojson` files; `**\/*.geojson` walks the tree recursively.
- *   Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same
- *   record, not new records.
- *
- *   Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair:
- *
- *   - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized
- *       variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`,
- *       `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case:
- *       `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become
- *       training rows for the same WOF id.
- *   - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2,
- *       country → 1, county → 1.
- *
- *   `source_id` is `wof-admin-<wof_id>-<name-slot>-<hierarchy-variant>`. The previous SQLite adapter
- *   used `wof-admin-<wof_id>-<hierarchy-variant>` (no name slot); the new format adds a name-slot
- *   segment so the colloquial / preferred / per-locale variants survive dedup independently.
- *
- *   License: CC0. The adapter stamps every row with `CC0-1.0`.
- */
-
-import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
-import type { ComponentTag } from "@mailwoman/core/types"
-
-import {
-	COUNTRY_DISPLAY_NAME,
-	emitWOFJSONRows,
-	LOCALE_BY_COUNTRY,
-	nameSlotsFor as wofNameSlotsFor,
-	type WOFVariantSpec,
-} from "#adapters/wof-json-rows"
-import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "#types"
-import { buildAncestryIndex, walkFeatures, type WOFRecord } from "#utils"
-
-/**
- * Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip.
- *
- * Per-adapter deliberately (the postalcode adapter carries its own): each table is a record FILTER for its adapter's
- * emission set, not a shared vocabulary.
- */
-function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
-	switch (placetype) {
-		case "country":
-		case "nation":
-			return "country"
-		case "macroregion":
-		case "region":
-			return "region"
-		case "macrocounty":
-		case "county":
-		case "localadmin":
-			return "subregion"
-		case "locality":
-			return "locality"
-		case "borough":
-		case "macrohood":
-		case "neighbourhood":
-		case "microhood":
-			return "dependent_locality"
-		default:
-			return undefined
-	}
-}
-
-/**
- * Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
- *
- * `selfName` is the surface form to use for the record's own component (locality / region / country / subregion).
- * Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` localized value for variant slots;
- * ancestor names always come from the ancestor's canonical `wof:name`.
- *
- * Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
- * canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
- */
-export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[] {
-	const selfTag = placetypeToTag(row.placetype)
-
-	if (!selfTag) return []
-
-	const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
-	const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
-	const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
-
-	const variants: WOFVariantSpec[] = []
-
-	switch (selfTag) {
-		case "locality":
-		case "dependent_locality": {
-			variants.push({ suffix: "self", components: { [selfTag]: selfName } })
-
-			if (region) {
-				variants.push({
-					suffix: "with-region",
-					components: { [selfTag]: selfName, region: region.name },
-				})
-			}
-
-			if (region && country) {
-				variants.push({
-					suffix: "with-region-country",
-					components: { [selfTag]: selfName, region: region.name, country: countryDisplay },
-				})
-			} else if (!region && country) {
-				variants.push({
-					suffix: "with-country",
-					components: { [selfTag]: selfName, country: countryDisplay },
-				})
-			}
-
-			return variants
-		}
-
-		case "region": {
-			variants.push({ suffix: "self", components: { region: selfName } })
-
-			if (country) {
-				variants.push({
-					suffix: "with-country",
-					components: { region: selfName, country: countryDisplay },
-				})
-			}
-
-			return variants
-		}
-
-		case "country": {
-			variants.push({ suffix: "self", components: { country: selfName } })
-
-			return variants
-		}
-
-		case "subregion": {
-			variants.push({ suffix: "self", components: { subregion: selfName } })
-
-			return variants
-		}
-
-		default:
-			return []
-	}
-}
-
-/**
- * Build the per-record name-slot list. The canonical `"default"` slot uses the OpenCage-canonical country form when the
- * record is itself a country (matches SQLite-adapter behavior); every other placetype's default slot uses `wof:name`
- * verbatim.
- */
-export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
-	return wofNameSlotsFor(rec, {
-		canonicalName: (record) =>
-			placetypeToTag(record.placetype) === "country"
-				? (COUNTRY_DISPLAY_NAME[record.country] ?? record.name)
-				: record.name,
-	})
-}
-
-/**
- * Registry id for this adapter. Stamped into every row it emits, so a corpus record can be traced back to the dataset
- * it came from.
- */
-export const WOF_ADMIN_ADAPTER_ID = "wof-admin"
-
-/**
- * Construct the wof-admin JSON-bundle adapter. The adapter is stateless across runs; calling this twice with the same
- * input directory produces byte-identical `canonical.jsonl` (records are emitted in sorted `wof:id` order to be
- * insensitive to filesystem walk ordering).
- */
-export function createWOFAdminAdapter(): CorpusAdapter {
-	return {
-		id: WOF_ADMIN_ADAPTER_ID,
-		defaultLicense: "CC0-1.0",
-		description:
-			"Who's On First admin GeoJSON bundles (countries, regions, counties, localities) — multi-name variants per record.",
-
-		async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
-			// Pass 1: scan every GeoJSON file once, build the in-memory record index.
-			// We keep only records whose placetype maps to a ComponentTag — irrelevant placetypes
-			// (campus, county-region hybrids on which Mailwoman has no opinion) are dropped here so
-			// they don't inflate the ancestry index. Country-filtered runs prune to the matching
-			// country code too; the ancestors of a same-country record live in the same admin repo.
-			const byID = new Map<number, WOFRecord>()
-
-			for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
-				if (opts.signal?.aborted) return
-
-				if (opts.country && rec.country !== opts.country) continue
-
-				if (!placetypeToTag(rec.placetype)) continue
-				byID.set(rec.id, rec)
-			}
-
-			const ancestry = buildAncestryIndex(byID)
-
-			// Pass 2: emit rows in sorted-id order for deterministic JSONL.
-			yield* emitWOFJSONRows({
-				records: byID,
-				ancestry,
-				adapterOptions: opts,
-				adapterID: WOF_ADMIN_ADAPTER_ID,
-				localeByCountry: LOCALE_BY_COUNTRY,
-				nameSlotsFor,
-				variantsFor,
-			})
-		},
-	}
-}
-
-/**
- * Single shared instance, suitable for `defaultAdapterRegistry`.
- */
-export const wofAdminAdapter = createWOFAdminAdapter()

package/lib/adapters/wof-json-rows.ts

@@ -1,129 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- * @file Deterministic canonical-row emission shared by WOF GeoJSON adapters.
- */
-
-import type { ComponentTag } from "@mailwoman/core/types"
-import { formatAddress, reconcileComponents } from "@mailwoman/formatter"
-
-import type { AdapterOptions, CanonicalRow } from "#types"
-import { normalizeNameKey, type WOFRecord } from "#utils"
-
-/**
- * Display name for the country, keyed by ISO 3166-1 alpha-2.
- *
- * Must be the **OpenCage-canonical** surface form: the `address-formatter` library expands some country names en route
- * to its output (e.g. `"United States"` → `"United States of America"`). If `components.country` and the formatted
- * `raw` disagree, alignment will fail downstream. Keying off the canonical form keeps the two in lockstep.
- *
- * Phase 1 US + FR only; extend as new locales come online. Missing countries fall back to the country row's `wof:name`,
- * accepting the alignment risk for non-canonicalized names.
- */
-export const COUNTRY_DISPLAY_NAME: Record<string, string> = {
-	US: "United States of America",
-	FR: "France",
-}
-
-/**
- * BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country.
- */
-export const LOCALE_BY_COUNTRY: Record<string, string> = {
-	US: "en-US",
-	FR: "fr-FR",
-}
-
-export interface WOFVariantSpec {
-	suffix: string
-	components: Partial<Record<ComponentTag, string>>
-}
-
-export interface NameSlotOptions {
-	/**
-	 * Canonical surface for the record's own `"default"` slot. Default `rec.name` verbatim; the admin adapter substitutes
-	 * the OpenCage-canonical {@link COUNTRY_DISPLAY_NAME} for country records.
-	 */
-	canonicalName?: (rec: WOFRecord) => string
-}
-
-/**
- * Build the per-record name-slot list: the canonical `"default"` slot, then every `name:*` variant deduplicated against
- * it so a redundant `"default"`-equivalent row is not emitted under a localized key.
- */
-export function nameSlotsFor(rec: WOFRecord, options: NameSlotOptions = {}): Array<{ key: string; value: string }> {
-	const canonicalSelfName = options.canonicalName?.(rec) ?? rec.name
-
-	const seen = new Set<string>([canonicalSelfName])
-	const slots: Array<{ key: string; value: string }> = [{ key: "default", value: canonicalSelfName }]
-
-	for (const [rawKey, value] of rec.nameVariants) {
-		if (seen.has(value)) continue
-		seen.add(value)
-		slots.push({ key: normalizeNameKey(rawKey), value })
-	}
-
-	return slots
-}
-
-interface EmitWOFJSONRowsOptions {
-	records: ReadonlyMap<number, WOFRecord>
-	ancestry: ReadonlyMap<number, WOFRecord[]>
-	adapterOptions: AdapterOptions
-	adapterID: string
-	localeByCountry: Readonly<Record<string, string>>
-	shouldEmit?: (record: WOFRecord) => boolean
-	nameSlotsFor: (record: WOFRecord) => Array<{ key: string; value: string }>
-	variantsFor: (record: WOFRecord, ancestry: WOFRecord[], selfName: string) => WOFVariantSpec[]
-}
-
-/**
- * Emit aligned rows in WOF-id order, enforcing the adapter limit across all name and hierarchy variants.
- */
-export function* emitWOFJSONRows(options: EmitWOFJSONRowsOptions): Generator<CanonicalRow> {
-	const {
-		records,
-		ancestry,
-		adapterOptions,
-		adapterID,
-		localeByCountry,
-		shouldEmit = () => true,
-		nameSlotsFor: slotsForRecord,
-		variantsFor,
-	} = options
-
-	let emitted = 0
-
-	for (const id of [...records.keys()].toSorted((a, b) => a - b)) {
-		if (adapterOptions.signal?.aborted) return
-		const record = records.get(id)!
-
-		if (!shouldEmit(record)) continue
-
-		for (const slot of slotsForRecord(record)) {
-			for (const variant of variantsFor(record, ancestry.get(id) ?? [], slot.value)) {
-				if (adapterOptions.limit !== undefined && emitted >= adapterOptions.limit) return
-
-				const raw = formatAddress(variant.components, record.country, { separator: ", " })
-
-				if (!raw) continue
-				const components = reconcileComponents(variant.components, raw)
-
-				if (!Object.keys(components).length) continue
-
-				yield {
-					raw,
-					components,
-					country: record.country,
-					locale: localeByCountry[record.country],
-					source: adapterID,
-					source_id: `${adapterID}-${record.id}-${slot.key}-${variant.suffix}`,
-					corpus_version: "",
-					license: "CC0-1.0",
-				}
-
-				emitted++
-			}
-		}
-	}
-}