@mailwoman/corpus 7.2.0 → 7.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +59 -27
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,211 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `usgov-samhsa-treatment-locator`: SAMHSA Behavioral Health Treatment Services Locator CSV
7
+ * consumer.
8
+ *
9
+ * SAMHSA's Treatment Locator (`findtreatment.gov`) is the federal directory of substance-use and
10
+ * mental-health treatment facilities. The published CSV carries the facility name, an optional
11
+ * secondary name (typically the organizational parent), and the postal address quad split into
12
+ * primary + secondary street lines. Phase 1.6 §1.2 (#22) selects this source for the same reason
13
+ * it selects HRSA: facility names are hand-typed venue strings and the addresses pass through
14
+ * enough human + system hands to accumulate the suite-designator + sub-tenant chaos ("Suite C,
15
+ * behind main building") that pure gazetteer data does not.
16
+ *
17
+ * SAMHSA's two-line address shape is the key adapter-specific concern. `street1` typically carries
18
+ * the canonical postal address (`"123 Main St"`); `street2` carries the suite / unit / "second
19
+ * floor" surface form. The adapter joins them with `", "` into a single `street` component (Phase
20
+ * 1 keeps `unit` as a deferred slot since the OpenCage template doesn't have a clean rendering
21
+ * for it). Operators wanting a different join policy can subclass the factory.
22
+ *
23
+ * Column names below match the canonical SAMHSA Behavioral Health Treatment Services Locator CSV
24
+ * export header. Operators substituting a closely-related extract should rename columns to match;
25
+ * the README has the mapping cheatsheet.
26
+ *
27
+ * License: stamped `"Public Domain"` per the SAMHSA Open Data Foundry's federal-government
28
+ * distribution terms.
29
+ */
30
+
31
+ import { createReadStream } from "node:fs"
32
+
33
+ import { parse as csvParse } from "csv-parse"
34
+
35
+ import { stableSourceID } from "../../adapter.ts"
36
+ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
37
+ import { reconcileComponents } from "../../format.ts"
38
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
39
+
40
+ export const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator"
41
+ export const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain"
42
+
43
+ /**
44
+ * Subset of SAMHSA Treatment Locator CSV columns consulted by the adapter. Column names match the canonical SAMHSA Open
45
+ * Data Foundry export header. `name1` is the venue; `name2` is optional and folded into the venue when present.
46
+ */
47
+ interface SamhsaSiteRow {
48
+ name1: string
49
+ name2?: string
50
+ street1: string
51
+ street2?: string
52
+ city: string
53
+ state: string
54
+ zip: string
55
+ /** Optional. Falls back to `stableSourceID` derived from components when missing. */
56
+ frid?: string
57
+ }
58
+
59
+ const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
60
+
61
+ function splitAddress(address: string): { house_number?: string; street: string } | null {
62
+ const trimmed = address.trim()
63
+
64
+ if (!trimmed) return null
65
+ const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
66
+
67
+ if (m) return { house_number: m[1], street: m[2]!.trim() }
68
+
69
+ return { street: trimmed }
70
+ }
71
+
72
+ /**
73
+ * Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
74
+ * building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
75
+ * the `unit` component — see the file-level comment.
76
+ */
77
+ function joinTwoLineStreet(street1: string, street2: string | undefined): string {
78
+ const s1 = street1.trim()
79
+ const s2 = (street2 ?? "").trim()
80
+
81
+ if (!s1 && !s2) return ""
82
+
83
+ if (!s2) return s1
84
+
85
+ if (!s1) return s2
86
+
87
+ return `${s1}, ${s2}`
88
+ }
89
+
90
+ /**
91
+ * Combine `name1` + optional `name2` into a single venue surface form. SAMHSA conventions:
92
+ *
93
+ * - `name1` is the program / clinic name ("Mountain Plains Counseling Services").
94
+ * - `name2` is the parent organization ("Catholic Charities of Wyoming"), if any.
95
+ *
96
+ * Both render together as `"<name1> - <name2>"` when both are present — geocoder users typically type either form, so
97
+ * the model benefits from the joined surface.
98
+ */
99
+ function composeVenue(name1: string, name2: string | undefined): string {
100
+ const n1 = name1.trim()
101
+ const n2 = (name2 ?? "").trim()
102
+
103
+ if (!n1 && !n2) return ""
104
+
105
+ if (!n2) return n1
106
+
107
+ if (!n1) return n2
108
+
109
+ return `${n1} - ${n2}`
110
+ }
111
+
112
+ /** Same envelope-style format as HRSA: venue prefix, street body, city/state/zip suffix. */
113
+ function composeRaw(
114
+ venue: string,
115
+ house: string | undefined,
116
+ street: string,
117
+ city: string,
118
+ state: string,
119
+ postcode: string
120
+ ): string {
121
+ const streetPart = [house, street].filter(Boolean).join(" ").trim()
122
+ const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
123
+
124
+ return [venue.trim(), streetPart, cityPart].filter(Boolean).join(", ")
125
+ }
126
+
127
+ export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
128
+ return {
129
+ id: USGOV_SAMHSA_ADAPTER_ID,
130
+ defaultLicense: USGOV_SAMHSA_DEFAULT_LICENSE,
131
+ description:
132
+ "SAMHSA Behavioral Health Treatment Services Locator (public-domain). Adversarial source: venue + two-line address co-occurrence, hand-entered.",
133
+
134
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
135
+ if (opts.country && opts.country !== "US") {
136
+ throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
137
+ }
138
+
139
+ const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
140
+ const parser = stream.pipe(
141
+ csvParse({
142
+ columns: true,
143
+ skip_empty_lines: true,
144
+ relax_quotes: true,
145
+ relax_column_count: true,
146
+ })
147
+ )
148
+
149
+ let emitted = 0
150
+
151
+ try {
152
+ for await (const record of parser as AsyncIterable<SamhsaSiteRow>) {
153
+ if (opts.signal?.aborted) break
154
+
155
+ if (opts.limit !== undefined && emitted >= opts.limit) break
156
+
157
+ const venue = composeVenue(record.name1 ?? "", record.name2)
158
+ const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
159
+ const split = splitAddress(street)
160
+ const city = (record.city ?? "").trim()
161
+ const stateAbbr = (record.state ?? "").trim()
162
+ const postcode = (record.zip ?? "").trim()
163
+
164
+ if (!venue || !split || !city || !postcode) continue
165
+ const state = lookupStateAbbreviation(stateAbbr)
166
+
167
+ if (!state) continue
168
+
169
+ // venue first — same kryptonite-defending insertion order as HRSA.
170
+ const components: CanonicalRow["components"] = {
171
+ venue,
172
+ ...(split.house_number ? { house_number: split.house_number } : {}),
173
+ street: split.street,
174
+ locality: city,
175
+ region: state.abbreviation,
176
+ postcode,
177
+ }
178
+
179
+ const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
180
+
181
+ if (!raw) continue
182
+
183
+ const aligned = reconcileComponents(components, raw)
184
+
185
+ if (Object.keys(aligned).length === 0) continue
186
+
187
+ const frID = (record.frid ?? "").trim()
188
+ const sourceID = frID
189
+ ? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}`
190
+ : stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
191
+
192
+ yield {
193
+ raw,
194
+ components: aligned,
195
+ country: "US",
196
+ locale: "en-US",
197
+ source: USGOV_SAMHSA_ADAPTER_ID,
198
+ source_id: sourceID,
199
+ corpus_version: "",
200
+ license: USGOV_SAMHSA_DEFAULT_LICENSE,
201
+ }
202
+ emitted++
203
+ }
204
+ } finally {
205
+ stream.destroy()
206
+ }
207
+ },
208
+ }
209
+ }
210
+
211
+ export const usgovSamhsaTreatmentLocatorAdapter = createUsgovSamhsaTreatmentLocatorAdapter()
@@ -0,0 +1,176 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `wof-admin-jp`: Japanese admin-hierarchy adapter.
7
+ *
8
+ * PROTOTYPE — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
9
+ * synthesizes BIO-labeled training rows from the global WOF SQLite.
10
+ *
11
+ * JP addresses differ from US/EU in three ways:
12
+ *
13
+ * 1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
14
+ * "1-2-3 Minamicho, Setagaya, Tokyo"
15
+ * 2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighbourhood"
16
+ * placetype (丁目) is the closest analog to a street but is actually a grid cell.
17
+ * 3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-NNNN`.
18
+ *
19
+ * Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
20
+ *
21
+ * | JP concept | WOF placetype | ComponentTag (Phase 6) |
22
+ * |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
23
+ * region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
24
+ * 丁目 (chome) | neighbourhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
25
+ * (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
26
+ *
27
+ * This adapter currently emits only the admin chain (region → locality → block). House numbers and
28
+ * sub-blocks require a different data source (JP postcode database or real-world address dumps
29
+ * from MLIT/JapanPost).
30
+ */
31
+
32
+ import { DatabaseSync } from "node:sqlite"
33
+
34
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
35
+
36
+ export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp"
37
+
38
+ interface PlaceRow {
39
+ id: number
40
+ name: string
41
+ placetype: string
42
+ parent_id: number
43
+ country: string
44
+ }
45
+
46
+ interface NameRow {
47
+ id: number
48
+ name: string
49
+ language: string
50
+ }
51
+
52
+ /** Walk parent chain up to 6 levels. */
53
+ function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, string>): PlaceRow[] {
54
+ const stmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
55
+ const out: PlaceRow[] = []
56
+ let id = startID
57
+
58
+ for (let i = 0; i < 6 && id > 0; i++) {
59
+ const row = stmt.get(id) as PlaceRow | undefined
60
+
61
+ if (!row) break
62
+ out.push(row)
63
+ id = row.parent_id
64
+ }
65
+
66
+ return out
67
+ }
68
+
69
+ /** Pick the best display name for a place: prefer Japanese variant, fall back to English. */
70
+ function pickName(row: PlaceRow, jpnNames: Map<number, string>): string {
71
+ return jpnNames.get(row.id) ?? row.name
72
+ }
73
+
74
+ /**
75
+ * Synthesize a JP address from a hierarchy chain.
76
+ *
77
+ * Format: `〒<postcode>? <region><locality><neighbourhood>?`
78
+ *
79
+ * No house numbers yet — needs MLIT data.
80
+ */
81
+ export function synthesizeJpAddress(
82
+ chain: PlaceRow[],
83
+ jpnNames: Map<number, string>
84
+ ): {
85
+ raw: string
86
+ components: CanonicalRow["components"]
87
+ } | null {
88
+ const region = chain.find((r) => r.placetype === "region")
89
+ const locality = chain.find((r) => r.placetype === "locality" || r.placetype === "county")
90
+
91
+ if (!region || !locality) return null
92
+
93
+ const neighbourhood = chain.find((r) => r.placetype === "neighbourhood")
94
+
95
+ const regionName = pickName(region, jpnNames)
96
+ const localityName = pickName(locality, jpnNames)
97
+ const neighbourhoodName = neighbourhood ? pickName(neighbourhood, jpnNames) : null
98
+
99
+ const components: CanonicalRow["components"] = {
100
+ region: regionName,
101
+ locality: localityName,
102
+ country: "JP",
103
+ }
104
+
105
+ if (neighbourhoodName) {
106
+ components.dependent_locality = neighbourhoodName
107
+ }
108
+
109
+ const raw = [regionName, localityName, neighbourhoodName].filter(Boolean).join("")
110
+
111
+ return { raw, components }
112
+ }
113
+
114
+ /**
115
+ * Build the JP adapter. Reads from the unified global WOF SQLite, walks admin chains starting from neighbourhoods, and
116
+ * yields canonical rows.
117
+ */
118
+ export function createWOFAdminJpAdapter(): CorpusAdapter {
119
+ return {
120
+ id: WOF_ADMIN_JP_ADAPTER_ID,
121
+ defaultLicense: "CC-BY-4.0",
122
+ description: "Japanese admin hierarchy from WOF (synthesized addresses without house numbers).",
123
+
124
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
125
+ if (opts.country && opts.country !== "JP") {
126
+ throw new Error(`wof-admin-jp adapter: only JP supported, got country=${opts.country}`)
127
+ }
128
+
129
+ const db = new DatabaseSync(opts.inputPath, { readOnly: true })
130
+
131
+ try {
132
+ const jpnNamesStmt = db.prepare(`SELECT id, name FROM names WHERE language = 'jpn'`)
133
+ const jpnNames = new Map<number, string>()
134
+
135
+ for (const row of jpnNamesStmt.all() as { id: number; name: string }[]) {
136
+ if (!jpnNames.has(row.id)) {
137
+ jpnNames.set(row.id, row.name)
138
+ }
139
+ }
140
+
141
+ const seeds = db.prepare(`SELECT id FROM spr WHERE country='JP' AND placetype='neighbourhood'`).all() as {
142
+ id: number
143
+ }[]
144
+
145
+ let emitted = 0
146
+
147
+ for (const seed of seeds) {
148
+ if (opts.signal?.aborted) break
149
+
150
+ if (opts.limit !== undefined && emitted >= opts.limit) break
151
+
152
+ const chain = chainOf(db, seed.id, jpnNames)
153
+ const synth = synthesizeJpAddress(chain, jpnNames)
154
+
155
+ if (!synth) continue
156
+
157
+ yield {
158
+ raw: synth.raw,
159
+ components: synth.components,
160
+ country: "JP",
161
+ locale: "ja-JP",
162
+ source: WOF_ADMIN_JP_ADAPTER_ID,
163
+ source_id: `${WOF_ADMIN_JP_ADAPTER_ID}-${seed.id}`,
164
+ corpus_version: "",
165
+ license: "CC-BY-4.0",
166
+ }
167
+ emitted++
168
+ }
169
+ } finally {
170
+ db.close()
171
+ }
172
+ },
173
+ }
174
+ }
175
+
176
+ export const wofAdminJpAdapter = createWOFAdminJpAdapter()
@@ -0,0 +1,280 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `wof-admin`: Who's On First admin GeoJSON-bundle adapter.
7
+ *
8
+ * **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at
9
+ * `packages/corpus/src/adapters/wof-admin/`, removed in this same change) was replaced by this
10
+ * one because the SQLite distribution path was unworkable for the real corpus build:
11
+ *
12
+ * 1. `dist.whosonfirst.org/sqlite/` is dead (NXDOMAIN); the Geocode-Earth mirror is the only one.
13
+ * 2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated
14
+ * as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows.
15
+ * 3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a
16
+ * separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases
17
+ * (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a
18
+ * patched `is_current` predicate.
19
+ *
20
+ * Input: a directory containing one or more cloned `whosonfirst-data-admin-<cc>` GitHub repos. Each
21
+ * repo has `data/XXX/YYY/ZZZ/<wof-id>.geojson` files; `**\/*.geojson` walks the tree recursively.
22
+ * Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same
23
+ * record, not new records.
24
+ *
25
+ * Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair:
26
+ *
27
+ * - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized
28
+ * variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`,
29
+ * `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case:
30
+ * `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become
31
+ * training rows for the same WOF id.
32
+ * - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2,
33
+ * country → 1, county → 1.
34
+ *
35
+ * `source_id` is `wof-admin-<wof_id>-<name-slot>-<hierarchy-variant>`. The previous SQLite adapter
36
+ * used `wof-admin-<wof_id>-<hierarchy-variant>` (no name slot); the new format adds a name-slot
37
+ * segment so the colloquial / preferred / per-locale variants survive dedup independently.
38
+ *
39
+ * License: CC0. The adapter stamps every row with `CC0-1.0`.
40
+ */
41
+
42
+ import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
43
+ import type { ComponentTag } from "@mailwoman/core/types"
44
+
45
+ import { formatAddress, reconcileComponents } from "../../format.ts"
46
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
47
+ import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "../../wof-json.ts"
48
+
49
+ /**
50
+ * Display name for the country, keyed by ISO 3166-1 alpha-2.
51
+ *
52
+ * Must be the **OpenCage-canonical** surface form: the `address-formatter` library expands some country names en route
53
+ * to its output (e.g. `"United States"` → `"United States of America"`). If `components.country` and the formatted
54
+ * `raw` disagree, alignment will fail downstream. Keying off the canonical form keeps the two in lockstep.
55
+ *
56
+ * Phase 1 US + FR only; extend as new locales come online. Missing countries fall back to the country row's `wof:name`,
57
+ * accepting the alignment risk for non-canonicalized names.
58
+ */
59
+ const COUNTRY_DISPLAY_NAME: Record<string, string> = {
60
+ US: "United States of America",
61
+ FR: "France",
62
+ }
63
+
64
+ /** BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country. */
65
+ const LOCALE_BY_COUNTRY: Record<string, string> = {
66
+ US: "en-US",
67
+ FR: "fr-FR",
68
+ }
69
+
70
+ /** Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip. */
71
+ function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
72
+ switch (placetype) {
73
+ case "country":
74
+ case "nation":
75
+ return "country"
76
+ case "macroregion":
77
+ case "region":
78
+ return "region"
79
+ case "macrocounty":
80
+ case "county":
81
+ case "localadmin":
82
+ return "subregion"
83
+ case "locality":
84
+ return "locality"
85
+ case "borough":
86
+ case "macrohood":
87
+ case "neighbourhood":
88
+ case "microhood":
89
+ return "dependent_locality"
90
+ default:
91
+ return undefined
92
+ }
93
+ }
94
+
95
+ interface VariantSpec {
96
+ /** Hierarchy-variant id appended to `source_id`. */
97
+ suffix: string
98
+
99
+ /** Component tag → display string the adapter will hand to the runner. */
100
+ components: Partial<Record<ComponentTag, string>>
101
+ }
102
+
103
+ /**
104
+ * Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
105
+ *
106
+ * `selfName` is the surface form to use for the record's own component (locality / region / country / subregion).
107
+ * Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` localized value for variant slots;
108
+ * ancestor names always come from the ancestor's canonical `wof:name`.
109
+ *
110
+ * Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
111
+ * canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
112
+ */
113
+ export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): VariantSpec[] {
114
+ const selfTag = placetypeToTag(row.placetype)
115
+
116
+ if (!selfTag) return []
117
+
118
+ const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
119
+ const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
120
+ const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
121
+
122
+ const variants: VariantSpec[] = []
123
+
124
+ switch (selfTag) {
125
+ case "locality":
126
+ case "dependent_locality": {
127
+ variants.push({ suffix: "self", components: { [selfTag]: selfName } })
128
+
129
+ if (region) {
130
+ variants.push({
131
+ suffix: "with-region",
132
+ components: { [selfTag]: selfName, region: region.name },
133
+ })
134
+ }
135
+
136
+ if (region && country) {
137
+ variants.push({
138
+ suffix: "with-region-country",
139
+ components: { [selfTag]: selfName, region: region.name, country: countryDisplay },
140
+ })
141
+ } else if (!region && country) {
142
+ variants.push({
143
+ suffix: "with-country",
144
+ components: { [selfTag]: selfName, country: countryDisplay },
145
+ })
146
+ }
147
+
148
+ return variants
149
+ }
150
+
151
+ case "region": {
152
+ variants.push({ suffix: "self", components: { region: selfName } })
153
+
154
+ if (country) {
155
+ variants.push({
156
+ suffix: "with-country",
157
+ components: { region: selfName, country: countryDisplay },
158
+ })
159
+ }
160
+
161
+ return variants
162
+ }
163
+
164
+ case "country": {
165
+ variants.push({ suffix: "self", components: { country: selfName } })
166
+
167
+ return variants
168
+ }
169
+
170
+ case "subregion": {
171
+ variants.push({ suffix: "self", components: { subregion: selfName } })
172
+
173
+ return variants
174
+ }
175
+
176
+ default:
177
+ return []
178
+ }
179
+ }
180
+
181
+ /**
182
+ * Build the per-record name-slot list. The canonical `"default"` slot uses the OpenCage-canonical country form when the
183
+ * record is itself a country (matches SQLite-adapter behavior); every other placetype's default slot uses `wof:name`
184
+ * verbatim.
185
+ *
186
+ * Subsequent slots come from `name:*` variants, deduplicated against the default name so we don't emit a redundant
187
+ * `"default"`-equivalent row under a localized key.
188
+ */
189
+ export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
190
+ const selfTag = placetypeToTag(rec.placetype)
191
+ const canonicalSelfName = selfTag === "country" ? (COUNTRY_DISPLAY_NAME[rec.country] ?? rec.name) : rec.name
192
+
193
+ const seen = new Set<string>([canonicalSelfName])
194
+ const slots: Array<{ key: string; value: string }> = [{ key: "default", value: canonicalSelfName }]
195
+
196
+ for (const [rawKey, value] of rec.nameVariants) {
197
+ if (seen.has(value)) continue
198
+ seen.add(value)
199
+ slots.push({ key: normalizeNameKey(rawKey), value })
200
+ }
201
+
202
+ return slots
203
+ }
204
+
205
+ export const WOF_ADMIN_ADAPTER_ID = "wof-admin"
206
+
207
+ /**
208
+ * Construct the wof-admin JSON-bundle adapter. The adapter is stateless across runs; calling this twice with the same
209
+ * input directory produces byte-identical `canonical.jsonl` (records are emitted in sorted `wof:id` order to be
210
+ * insensitive to filesystem walk ordering).
211
+ */
212
+ export function createWOFAdminAdapter(): CorpusAdapter {
213
+ return {
214
+ id: WOF_ADMIN_ADAPTER_ID,
215
+ defaultLicense: "CC0-1.0",
216
+ description:
217
+ "Who's On First admin GeoJSON bundles (countries, regions, counties, localities) — multi-name variants per record.",
218
+
219
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
220
+ // Pass 1: scan every GeoJSON file once, build the in-memory record index.
221
+ // We keep only records whose placetype maps to a ComponentTag — irrelevant placetypes
222
+ // (campus, county-region hybrids on which Mailwoman has no opinion) are dropped here so
223
+ // they don't inflate the ancestry index. Country-filtered runs prune to the matching
224
+ // country code too; the ancestors of a same-country record live in the same admin repo.
225
+ const byID = new Map<number, WOFRecord>()
226
+
227
+ for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
228
+ if (opts.signal?.aborted) return
229
+
230
+ if (opts.country && rec.country !== opts.country) continue
231
+
232
+ if (!placetypeToTag(rec.placetype)) continue
233
+ byID.set(rec.id, rec)
234
+ }
235
+
236
+ const ancestry = buildAncestryIndex(byID)
237
+
238
+ // Pass 2: emit rows in sorted-id order for deterministic JSONL.
239
+ const ids = [...byID.keys()].sort((a, b) => a - b)
240
+ let emitted = 0
241
+
242
+ for (const id of ids) {
243
+ if (opts.signal?.aborted) return
244
+ const rec = byID.get(id)!
245
+ const chain = ancestry.get(id) ?? []
246
+ const slots = nameSlotsFor(rec)
247
+
248
+ for (const slot of slots) {
249
+ const variants = variantsFor(rec, chain, slot.value)
250
+
251
+ for (const variant of variants) {
252
+ if (opts.limit !== undefined && emitted >= opts.limit) return
253
+
254
+ const raw = formatAddress(variant.components, rec.country, { separator: ", " })
255
+
256
+ if (!raw) continue
257
+ const aligned = reconcileComponents(variant.components, raw)
258
+
259
+ if (Object.keys(aligned).length === 0) continue
260
+
261
+ yield {
262
+ raw,
263
+ components: aligned,
264
+ country: rec.country,
265
+ locale: LOCALE_BY_COUNTRY[rec.country],
266
+ source: WOF_ADMIN_ADAPTER_ID,
267
+ source_id: `${WOF_ADMIN_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
268
+ corpus_version: "",
269
+ license: "CC0-1.0",
270
+ }
271
+ emitted++
272
+ }
273
+ }
274
+ }
275
+ },
276
+ }
277
+ }
278
+
279
+ /** Single shared instance, suitable for `defaultAdapterRegistry`. */
280
+ export const wofAdminAdapter = createWOFAdminAdapter()