@mailwoman/corpus 7.2.0 → 7.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +59 -27
- package/src/adapter.ts +153 -0
- package/src/adapters/ban/adapter.ts +189 -0
- package/src/adapters/ban/street-decompose.ts +98 -0
- package/src/adapters/fcc-bdc/adapter.ts +188 -0
- package/src/adapters/geonames/adapter.ts +185 -0
- package/src/adapters/geonames-postal/adapter.ts +111 -0
- package/src/adapters/gnaf/adapter.ts +139 -0
- package/src/adapters/gnaf/assemble.ts +200 -0
- package/src/adapters/index.ts +135 -0
- package/src/adapters/openaddresses/adapter.ts +237 -0
- package/src/adapters/overture/adapter.ts +149 -0
- package/src/adapters/state-hi-schools/adapter.ts +171 -0
- package/src/adapters/state-ia-contractors/adapter.ts +148 -0
- package/src/adapters/state-ny-notaries/adapter.ts +169 -0
- package/src/adapters/state-tx-notaries/adapter.ts +160 -0
- package/src/adapters/synth-po-box/adapter.ts +197 -0
- package/src/adapters/tiger/adapter.ts +232 -0
- package/src/adapters/tiger/street-decompose.ts +120 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
- package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
- package/src/adapters/usgov-nad/adapter.ts +338 -0
- package/src/adapters/usgov-nppes/adapter.ts +169 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
- package/src/adapters/wof-admin-jp/adapter.ts +176 -0
- package/src/adapters/wof-admin-json/adapter.ts +280 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
- package/src/align.ts +317 -0
- package/src/build.ts +348 -0
- package/src/codex/us-fips-state.ts +109 -0
- package/src/format.ts +14 -0
- package/src/golden.ts +147 -0
- package/src/index.ts +28 -0
- package/src/license.ts +48 -0
- package/src/parquet-wrapper/index.ts +9 -0
- package/src/parquet-wrapper/reader.ts +67 -0
- package/src/parquet-wrapper/schema.ts +80 -0
- package/src/parquet-wrapper/writer.ts +102 -0
- package/src/parquet.ts +360 -0
- package/src/runner.ts +283 -0
- package/src/shard-recipes/anchor-absorption.ts +86 -0
- package/src/shard-recipes/boundary-stress.ts +87 -0
- package/src/shard-recipes/country-balanced.ts +471 -0
- package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
- package/src/shard-recipes/fr-admin-split.ts +212 -0
- package/src/shard-recipes/fr-bare-street.ts +85 -0
- package/src/shard-recipes/fr-fragment.ts +304 -0
- package/src/shard-recipes/fr-order.ts +283 -0
- package/src/shard-recipes/german.ts +206 -0
- package/src/shard-recipes/house-venue.ts +78 -0
- package/src/shard-recipes/index.ts +76 -0
- package/src/shard-recipes/intersection.ts +617 -0
- package/src/shard-recipes/locale.ts +415 -0
- package/src/shard-recipes/nl-postcode.ts +105 -0
- package/src/shard-recipes/no-fragment.ts +206 -0
- package/src/shard-recipes/no-street-led.ts +130 -0
- package/src/shard-recipes/no-street.ts +78 -0
- package/src/shard-recipes/po-box-cedex.ts +853 -0
- package/src/shard-recipes/po-box.ts +111 -0
- package/src/shard-recipes/scaffold.ts +187 -0
- package/src/shard-recipes/si-bare-village.ts +88 -0
- package/src/shard-recipes/street-affix.ts +549 -0
- package/src/shard-recipes/street-bare.ts +90 -0
- package/src/shard-recipes/street.ts +74 -0
- package/src/shard-recipes/unit.ts +315 -0
- package/src/split.ts +251 -0
- package/src/synthesize-anchor-absorption.ts +263 -0
- package/src/synthesize-boundary-stress.ts +504 -0
- package/src/synthesize-german.ts +227 -0
- package/src/synthesize-house-venue.ts +192 -0
- package/src/synthesize-intersection.ts +205 -0
- package/src/synthesize-no-street.ts +341 -0
- package/src/synthesize-po-box.ts +307 -0
- package/src/synthesize-street.ts +264 -0
- package/src/synthesize.ts +872 -0
- package/src/tokenize.ts +68 -0
- package/src/tools/align-shard.ts +67 -0
- package/src/tools/audit.ts +352 -0
- package/src/tools/corpus-stats.ts +183 -0
- package/src/tools/fetch/ban.ts +288 -0
- package/src/tools/fetch/download.ts +119 -0
- package/src/tools/fetch/hrsa.ts +65 -0
- package/src/tools/fetch/imls-pls.ts +162 -0
- package/src/tools/fetch/index.ts +121 -0
- package/src/tools/fetch/nad.ts +304 -0
- package/src/tools/fetch/nppes.ts +177 -0
- package/src/tools/fetch/openaddresses.ts +380 -0
- package/src/tools/fetch/state-hi-schools.ts +217 -0
- package/src/tools/fetch/state-sources.ts +142 -0
- package/src/tools/fetch/tiger-full.ts +318 -0
- package/src/tools/golden-expand.ts +524 -0
- package/src/tools/golden-promote.ts +304 -0
- package/src/tools/index.ts +24 -0
- package/src/tools/ingest-csv.ts +420 -0
- package/src/tools/jsonl-to-parquet.ts +200 -0
- package/src/tools/lint-shard-vocab.ts +357 -0
- package/src/tools/lint-shard.ts +502 -0
- package/src/tools/overlay-manifest.ts +141 -0
- package/src/tools/shard-kryptonite.ts +131 -0
- package/src/tools/shard-translit.ts +244 -0
- package/src/types.ts +213 -0
- package/src/wof-json.ts +236 -0
|
@@ -0,0 +1,211 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `usgov-samhsa-treatment-locator`: SAMHSA Behavioral Health Treatment Services Locator CSV
|
|
7
|
+
* consumer.
|
|
8
|
+
*
|
|
9
|
+
* SAMHSA's Treatment Locator (`findtreatment.gov`) is the federal directory of substance-use and
|
|
10
|
+
* mental-health treatment facilities. The published CSV carries the facility name, an optional
|
|
11
|
+
* secondary name (typically the organizational parent), and the postal address quad split into
|
|
12
|
+
* primary + secondary street lines. Phase 1.6 §1.2 (#22) selects this source for the same reason
|
|
13
|
+
* it selects HRSA: facility names are hand-typed venue strings and the addresses pass through
|
|
14
|
+
* enough human + system hands to accumulate the suite-designator + sub-tenant chaos ("Suite C,
|
|
15
|
+
* behind main building") that pure gazetteer data does not.
|
|
16
|
+
*
|
|
17
|
+
* SAMHSA's two-line address shape is the key adapter-specific concern. `street1` typically carries
|
|
18
|
+
* the canonical postal address (`"123 Main St"`); `street2` carries the suite / unit / "second
|
|
19
|
+
* floor" surface form. The adapter joins them with `", "` into a single `street` component (Phase
|
|
20
|
+
* 1 keeps `unit` as a deferred slot since the OpenCage template doesn't have a clean rendering
|
|
21
|
+
* for it). Operators wanting a different join policy can subclass the factory.
|
|
22
|
+
*
|
|
23
|
+
* Column names below match the canonical SAMHSA Behavioral Health Treatment Services Locator CSV
|
|
24
|
+
* export header. Operators substituting a closely-related extract should rename columns to match;
|
|
25
|
+
* the README has the mapping cheatsheet.
|
|
26
|
+
*
|
|
27
|
+
* License: stamped `"Public Domain"` per the SAMHSA Open Data Foundry's federal-government
|
|
28
|
+
* distribution terms.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import { createReadStream } from "node:fs"
|
|
32
|
+
|
|
33
|
+
import { parse as csvParse } from "csv-parse"
|
|
34
|
+
|
|
35
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
36
|
+
import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
37
|
+
import { reconcileComponents } from "../../format.ts"
|
|
38
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
39
|
+
|
|
40
|
+
export const USGOV_SAMHSA_ADAPTER_ID = "usgov-samhsa-treatment-locator"
|
|
41
|
+
export const USGOV_SAMHSA_DEFAULT_LICENSE = "Public Domain"
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Subset of SAMHSA Treatment Locator CSV columns consulted by the adapter. Column names match the canonical SAMHSA Open
|
|
45
|
+
* Data Foundry export header. `name1` is the venue; `name2` is optional and folded into the venue when present.
|
|
46
|
+
*/
|
|
47
|
+
interface SamhsaSiteRow {
|
|
48
|
+
name1: string
|
|
49
|
+
name2?: string
|
|
50
|
+
street1: string
|
|
51
|
+
street2?: string
|
|
52
|
+
city: string
|
|
53
|
+
state: string
|
|
54
|
+
zip: string
|
|
55
|
+
/** Optional. Falls back to `stableSourceID` derived from components when missing. */
|
|
56
|
+
frid?: string
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
60
|
+
|
|
61
|
+
function splitAddress(address: string): { house_number?: string; street: string } | null {
|
|
62
|
+
const trimmed = address.trim()
|
|
63
|
+
|
|
64
|
+
if (!trimmed) return null
|
|
65
|
+
const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
|
|
66
|
+
|
|
67
|
+
if (m) return { house_number: m[1], street: m[2]!.trim() }
|
|
68
|
+
|
|
69
|
+
return { street: trimmed }
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/**
|
|
73
|
+
* Join the SAMHSA two-line street: primary street + optional secondary line (suite / unit / floor / "behind main
|
|
74
|
+
* building") on `", "`. The combined value is the `street` component surface form. Phase 1 does not break this out into
|
|
75
|
+
* the `unit` component — see the file-level comment.
|
|
76
|
+
*/
|
|
77
|
+
function joinTwoLineStreet(street1: string, street2: string | undefined): string {
|
|
78
|
+
const s1 = street1.trim()
|
|
79
|
+
const s2 = (street2 ?? "").trim()
|
|
80
|
+
|
|
81
|
+
if (!s1 && !s2) return ""
|
|
82
|
+
|
|
83
|
+
if (!s2) return s1
|
|
84
|
+
|
|
85
|
+
if (!s1) return s2
|
|
86
|
+
|
|
87
|
+
return `${s1}, ${s2}`
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Combine `name1` + optional `name2` into a single venue surface form. SAMHSA conventions:
|
|
92
|
+
*
|
|
93
|
+
* - `name1` is the program / clinic name ("Mountain Plains Counseling Services").
|
|
94
|
+
* - `name2` is the parent organization ("Catholic Charities of Wyoming"), if any.
|
|
95
|
+
*
|
|
96
|
+
* Both render together as `"<name1> - <name2>"` when both are present — geocoder users typically type either form, so
|
|
97
|
+
* the model benefits from the joined surface.
|
|
98
|
+
*/
|
|
99
|
+
function composeVenue(name1: string, name2: string | undefined): string {
|
|
100
|
+
const n1 = name1.trim()
|
|
101
|
+
const n2 = (name2 ?? "").trim()
|
|
102
|
+
|
|
103
|
+
if (!n1 && !n2) return ""
|
|
104
|
+
|
|
105
|
+
if (!n2) return n1
|
|
106
|
+
|
|
107
|
+
if (!n1) return n2
|
|
108
|
+
|
|
109
|
+
return `${n1} - ${n2}`
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/** Same envelope-style format as HRSA: venue prefix, street body, city/state/zip suffix. */
|
|
113
|
+
function composeRaw(
|
|
114
|
+
venue: string,
|
|
115
|
+
house: string | undefined,
|
|
116
|
+
street: string,
|
|
117
|
+
city: string,
|
|
118
|
+
state: string,
|
|
119
|
+
postcode: string
|
|
120
|
+
): string {
|
|
121
|
+
const streetPart = [house, street].filter(Boolean).join(" ").trim()
|
|
122
|
+
const cityPart = [city.trim(), [state, postcode].filter(Boolean).join(" ").trim()].filter(Boolean).join(", ")
|
|
123
|
+
|
|
124
|
+
return [venue.trim(), streetPart, cityPart].filter(Boolean).join(", ")
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function createUsgovSamhsaTreatmentLocatorAdapter(): CorpusAdapter {
|
|
128
|
+
return {
|
|
129
|
+
id: USGOV_SAMHSA_ADAPTER_ID,
|
|
130
|
+
defaultLicense: USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
131
|
+
description:
|
|
132
|
+
"SAMHSA Behavioral Health Treatment Services Locator (public-domain). Adversarial source: venue + two-line address co-occurrence, hand-entered.",
|
|
133
|
+
|
|
134
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
135
|
+
if (opts.country && opts.country !== "US") {
|
|
136
|
+
throw new Error(`usgov-samhsa adapter: only US supported, got country=${opts.country}`)
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
140
|
+
const parser = stream.pipe(
|
|
141
|
+
csvParse({
|
|
142
|
+
columns: true,
|
|
143
|
+
skip_empty_lines: true,
|
|
144
|
+
relax_quotes: true,
|
|
145
|
+
relax_column_count: true,
|
|
146
|
+
})
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
let emitted = 0
|
|
150
|
+
|
|
151
|
+
try {
|
|
152
|
+
for await (const record of parser as AsyncIterable<SamhsaSiteRow>) {
|
|
153
|
+
if (opts.signal?.aborted) break
|
|
154
|
+
|
|
155
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
156
|
+
|
|
157
|
+
const venue = composeVenue(record.name1 ?? "", record.name2)
|
|
158
|
+
const street = joinTwoLineStreet(record.street1 ?? "", record.street2)
|
|
159
|
+
const split = splitAddress(street)
|
|
160
|
+
const city = (record.city ?? "").trim()
|
|
161
|
+
const stateAbbr = (record.state ?? "").trim()
|
|
162
|
+
const postcode = (record.zip ?? "").trim()
|
|
163
|
+
|
|
164
|
+
if (!venue || !split || !city || !postcode) continue
|
|
165
|
+
const state = lookupStateAbbreviation(stateAbbr)
|
|
166
|
+
|
|
167
|
+
if (!state) continue
|
|
168
|
+
|
|
169
|
+
// venue first — same kryptonite-defending insertion order as HRSA.
|
|
170
|
+
const components: CanonicalRow["components"] = {
|
|
171
|
+
venue,
|
|
172
|
+
...(split.house_number ? { house_number: split.house_number } : {}),
|
|
173
|
+
street: split.street,
|
|
174
|
+
locality: city,
|
|
175
|
+
region: state.abbreviation,
|
|
176
|
+
postcode,
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const raw = composeRaw(venue, split.house_number, split.street, city, state.abbreviation, postcode)
|
|
180
|
+
|
|
181
|
+
if (!raw) continue
|
|
182
|
+
|
|
183
|
+
const aligned = reconcileComponents(components, raw)
|
|
184
|
+
|
|
185
|
+
if (Object.keys(aligned).length === 0) continue
|
|
186
|
+
|
|
187
|
+
const frID = (record.frid ?? "").trim()
|
|
188
|
+
const sourceID = frID
|
|
189
|
+
? `${USGOV_SAMHSA_ADAPTER_ID}-${frID}`
|
|
190
|
+
: stableSourceID(USGOV_SAMHSA_ADAPTER_ID, aligned)
|
|
191
|
+
|
|
192
|
+
yield {
|
|
193
|
+
raw,
|
|
194
|
+
components: aligned,
|
|
195
|
+
country: "US",
|
|
196
|
+
locale: "en-US",
|
|
197
|
+
source: USGOV_SAMHSA_ADAPTER_ID,
|
|
198
|
+
source_id: sourceID,
|
|
199
|
+
corpus_version: "",
|
|
200
|
+
license: USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
201
|
+
}
|
|
202
|
+
emitted++
|
|
203
|
+
}
|
|
204
|
+
} finally {
|
|
205
|
+
stream.destroy()
|
|
206
|
+
}
|
|
207
|
+
},
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export const usgovSamhsaTreatmentLocatorAdapter = createUsgovSamhsaTreatmentLocatorAdapter()
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `wof-admin-jp`: Japanese admin-hierarchy adapter.
|
|
7
|
+
*
|
|
8
|
+
* PROTOTYPE — not yet wired into the Stage 3 training corpus. Demonstrates the JP address shape and
|
|
9
|
+
* synthesizes BIO-labeled training rows from the global WOF SQLite.
|
|
10
|
+
*
|
|
11
|
+
* JP addresses differ from US/EU in three ways:
|
|
12
|
+
*
|
|
13
|
+
* 1. **Reversed ordering** — region → county → locality → block → house_number "東京都世田谷区南町1-2-3" not
|
|
14
|
+
* "1-2-3 Minamicho, Setagaya, Tokyo"
|
|
15
|
+
* 2. **No street names** — most JP addresses use a block/sub-block grid system. The "neighbourhood"
|
|
16
|
+
* placetype (丁目) is the closest analog to a street but is actually a grid cell.
|
|
17
|
+
* 3. **Postcode-first convention** — addresses are often prefixed with `〒NNN-NNNN`.
|
|
18
|
+
*
|
|
19
|
+
* Schema mapping to ComponentTags (subset of Stage 3 + JP-specific Phase 6 tags):
|
|
20
|
+
*
|
|
21
|
+
* | JP concept | WOF placetype | ComponentTag (Phase 6) |
|
|
22
|
+
* |-----------------|--------------------|-----------------------| | 都道府県 (prefecture) | region |
|
|
23
|
+
* region (or prefecture) | | 市区町村 (city/ward) | county/locality | locality (or municipality) | |
|
|
24
|
+
* 丁目 (chome) | neighbourhood | block (Phase 6 tag) | | 番地 (banchi) | (synth) | sub_block | | 号
|
|
25
|
+
* (gō) | (synth) | house_number | | 〒 (postcode prefix) | — | postcode |
|
|
26
|
+
*
|
|
27
|
+
* This adapter currently emits only the admin chain (region → locality → block). House numbers and
|
|
28
|
+
* sub-blocks require a different data source (JP postcode database or real-world address dumps
|
|
29
|
+
* from MLIT/JapanPost).
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { DatabaseSync } from "node:sqlite"
|
|
33
|
+
|
|
34
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
35
|
+
|
|
36
|
+
export const WOF_ADMIN_JP_ADAPTER_ID = "wof-admin-jp"
|
|
37
|
+
|
|
38
|
+
interface PlaceRow {
|
|
39
|
+
id: number
|
|
40
|
+
name: string
|
|
41
|
+
placetype: string
|
|
42
|
+
parent_id: number
|
|
43
|
+
country: string
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
interface NameRow {
|
|
47
|
+
id: number
|
|
48
|
+
name: string
|
|
49
|
+
language: string
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Walk parent chain up to 6 levels. */
|
|
53
|
+
function chainOf(db: DatabaseSync, startID: number, _jpnNames: Map<number, string>): PlaceRow[] {
|
|
54
|
+
const stmt = db.prepare(`SELECT id, name, placetype, parent_id, country FROM spr WHERE id = ?`)
|
|
55
|
+
const out: PlaceRow[] = []
|
|
56
|
+
let id = startID
|
|
57
|
+
|
|
58
|
+
for (let i = 0; i < 6 && id > 0; i++) {
|
|
59
|
+
const row = stmt.get(id) as PlaceRow | undefined
|
|
60
|
+
|
|
61
|
+
if (!row) break
|
|
62
|
+
out.push(row)
|
|
63
|
+
id = row.parent_id
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return out
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Pick the best display name for a place: prefer Japanese variant, fall back to English. */
|
|
70
|
+
function pickName(row: PlaceRow, jpnNames: Map<number, string>): string {
|
|
71
|
+
return jpnNames.get(row.id) ?? row.name
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Synthesize a JP address from a hierarchy chain.
|
|
76
|
+
*
|
|
77
|
+
* Format: `〒<postcode>? <region><locality><neighbourhood>?`
|
|
78
|
+
*
|
|
79
|
+
* No house numbers yet — needs MLIT data.
|
|
80
|
+
*/
|
|
81
|
+
export function synthesizeJpAddress(
|
|
82
|
+
chain: PlaceRow[],
|
|
83
|
+
jpnNames: Map<number, string>
|
|
84
|
+
): {
|
|
85
|
+
raw: string
|
|
86
|
+
components: CanonicalRow["components"]
|
|
87
|
+
} | null {
|
|
88
|
+
const region = chain.find((r) => r.placetype === "region")
|
|
89
|
+
const locality = chain.find((r) => r.placetype === "locality" || r.placetype === "county")
|
|
90
|
+
|
|
91
|
+
if (!region || !locality) return null
|
|
92
|
+
|
|
93
|
+
const neighbourhood = chain.find((r) => r.placetype === "neighbourhood")
|
|
94
|
+
|
|
95
|
+
const regionName = pickName(region, jpnNames)
|
|
96
|
+
const localityName = pickName(locality, jpnNames)
|
|
97
|
+
const neighbourhoodName = neighbourhood ? pickName(neighbourhood, jpnNames) : null
|
|
98
|
+
|
|
99
|
+
const components: CanonicalRow["components"] = {
|
|
100
|
+
region: regionName,
|
|
101
|
+
locality: localityName,
|
|
102
|
+
country: "JP",
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (neighbourhoodName) {
|
|
106
|
+
components.dependent_locality = neighbourhoodName
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
const raw = [regionName, localityName, neighbourhoodName].filter(Boolean).join("")
|
|
110
|
+
|
|
111
|
+
return { raw, components }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Build the JP adapter. Reads from the unified global WOF SQLite, walks admin chains starting from neighbourhoods, and
|
|
116
|
+
* yields canonical rows.
|
|
117
|
+
*/
|
|
118
|
+
export function createWOFAdminJpAdapter(): CorpusAdapter {
|
|
119
|
+
return {
|
|
120
|
+
id: WOF_ADMIN_JP_ADAPTER_ID,
|
|
121
|
+
defaultLicense: "CC-BY-4.0",
|
|
122
|
+
description: "Japanese admin hierarchy from WOF (synthesized addresses without house numbers).",
|
|
123
|
+
|
|
124
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
125
|
+
if (opts.country && opts.country !== "JP") {
|
|
126
|
+
throw new Error(`wof-admin-jp adapter: only JP supported, got country=${opts.country}`)
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const db = new DatabaseSync(opts.inputPath, { readOnly: true })
|
|
130
|
+
|
|
131
|
+
try {
|
|
132
|
+
const jpnNamesStmt = db.prepare(`SELECT id, name FROM names WHERE language = 'jpn'`)
|
|
133
|
+
const jpnNames = new Map<number, string>()
|
|
134
|
+
|
|
135
|
+
for (const row of jpnNamesStmt.all() as { id: number; name: string }[]) {
|
|
136
|
+
if (!jpnNames.has(row.id)) {
|
|
137
|
+
jpnNames.set(row.id, row.name)
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
const seeds = db.prepare(`SELECT id FROM spr WHERE country='JP' AND placetype='neighbourhood'`).all() as {
|
|
142
|
+
id: number
|
|
143
|
+
}[]
|
|
144
|
+
|
|
145
|
+
let emitted = 0
|
|
146
|
+
|
|
147
|
+
for (const seed of seeds) {
|
|
148
|
+
if (opts.signal?.aborted) break
|
|
149
|
+
|
|
150
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
151
|
+
|
|
152
|
+
const chain = chainOf(db, seed.id, jpnNames)
|
|
153
|
+
const synth = synthesizeJpAddress(chain, jpnNames)
|
|
154
|
+
|
|
155
|
+
if (!synth) continue
|
|
156
|
+
|
|
157
|
+
yield {
|
|
158
|
+
raw: synth.raw,
|
|
159
|
+
components: synth.components,
|
|
160
|
+
country: "JP",
|
|
161
|
+
locale: "ja-JP",
|
|
162
|
+
source: WOF_ADMIN_JP_ADAPTER_ID,
|
|
163
|
+
source_id: `${WOF_ADMIN_JP_ADAPTER_ID}-${seed.id}`,
|
|
164
|
+
corpus_version: "",
|
|
165
|
+
license: "CC-BY-4.0",
|
|
166
|
+
}
|
|
167
|
+
emitted++
|
|
168
|
+
}
|
|
169
|
+
} finally {
|
|
170
|
+
db.close()
|
|
171
|
+
}
|
|
172
|
+
},
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
export const wofAdminJpAdapter = createWOFAdminJpAdapter()
|
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `wof-admin`: Who's On First admin GeoJSON-bundle adapter.
|
|
7
|
+
*
|
|
8
|
+
* **Phase 1.5.1 pivot.** The original Phase 1.5 SQLite adapter (formerly at
|
|
9
|
+
* `packages/corpus/src/adapters/wof-admin/`, removed in this same change) was replaced by this
|
|
10
|
+
* one because the SQLite distribution path was unworkable for the real corpus build:
|
|
11
|
+
*
|
|
12
|
+
* 1. `dist.whosonfirst.org/sqlite/` is dead (NXDOMAIN); the Geocode-Earth mirror is the only one.
|
|
13
|
+
* 2. The Geocode-Earth-hosted postalcode DB tags every row `mz:is_current = -1` ("unknown but treated
|
|
14
|
+
* as active"); the SQLite adapter's `is_current = 1` predicate emitted zero rows.
|
|
15
|
+
* 3. The `names` table in the SQLite distribution is empty — localized `name:*` variants live in a
|
|
16
|
+
* separate distribution. The St. Petersburg / Mt. Vernon / Ft. Lauderdale alternation cases
|
|
17
|
+
* (the original Phase 1.5.1 motivator) cannot be solved on the SQLite path even with a
|
|
18
|
+
* patched `is_current` predicate.
|
|
19
|
+
*
|
|
20
|
+
* Input: a directory containing one or more cloned `whosonfirst-data-admin-<cc>` GitHub repos. Each
|
|
21
|
+
* repo has `data/XXX/YYY/ZZZ/<wof-id>.geojson` files; `**\/*.geojson` walks the tree recursively.
|
|
22
|
+
* Alternate-geometry siblings (`-alt-*`) are skipped — they're separate exports of the same
|
|
23
|
+
* record, not new records.
|
|
24
|
+
*
|
|
25
|
+
* Per record, the adapter emits one row per `(name-variant, hierarchy-variant)` pair:
|
|
26
|
+
*
|
|
27
|
+
* - **Name variants**: the canonical `wof:name` (slot key `default`) plus every `name:*` localized
|
|
28
|
+
* variant present on the feature (`name:eng_x_preferred`, `name:eng_x_colloquial`,
|
|
29
|
+
* `name:rus_x_preferred`, ...). This is the Phase 1.5.1 fix for the St. Petersburg case:
|
|
30
|
+
* `"Saint Petersburg"` (canonical) and `"St. Petersburg"` (eng_x_colloquial) both become
|
|
31
|
+
* training rows for the same WOF id.
|
|
32
|
+
* - **Hierarchy variants** (unchanged from the SQLite adapter): locality → 3 variants, region → 2,
|
|
33
|
+
* country → 1, county → 1.
|
|
34
|
+
*
|
|
35
|
+
* `source_id` is `wof-admin-<wof_id>-<name-slot>-<hierarchy-variant>`. The previous SQLite adapter
|
|
36
|
+
* used `wof-admin-<wof_id>-<hierarchy-variant>` (no name slot); the new format adds a name-slot
|
|
37
|
+
* segment so the colloquial / preferred / per-locale variants survive dedup independently.
|
|
38
|
+
*
|
|
39
|
+
* License: CC0. The adapter stamps every row with `CC0-1.0`.
|
|
40
|
+
*/
|
|
41
|
+
|
|
42
|
+
import type { WhosOnFirstPlacetype } from "@mailwoman/core/resources/whosonfirst"
|
|
43
|
+
import type { ComponentTag } from "@mailwoman/core/types"
|
|
44
|
+
|
|
45
|
+
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
46
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
47
|
+
import { buildAncestryIndex, normalizeNameKey, walkFeatures, type WOFRecord } from "../../wof-json.ts"
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Display name for the country, keyed by ISO 3166-1 alpha-2.
|
|
51
|
+
*
|
|
52
|
+
* Must be the **OpenCage-canonical** surface form: the `address-formatter` library expands some country names en route
|
|
53
|
+
* to its output (e.g. `"United States"` → `"United States of America"`). If `components.country` and the formatted
|
|
54
|
+
* `raw` disagree, alignment will fail downstream. Keying off the canonical form keeps the two in lockstep.
|
|
55
|
+
*
|
|
56
|
+
* Phase 1 US + FR only; extend as new locales come online. Missing countries fall back to the country row's `wof:name`,
|
|
57
|
+
* accepting the alignment risk for non-canonicalized names.
|
|
58
|
+
*/
|
|
59
|
+
const COUNTRY_DISPLAY_NAME: Record<string, string> = {
|
|
60
|
+
US: "United States of America",
|
|
61
|
+
FR: "France",
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** BCP-47 locale defaulting for the corpus row's `locale` field. Defaulted by country. */
|
|
65
|
+
const LOCALE_BY_COUNTRY: Record<string, string> = {
|
|
66
|
+
US: "en-US",
|
|
67
|
+
FR: "fr-FR",
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** Map a WOF placetype to a Mailwoman `ComponentTag`, or `undefined` to skip. */
|
|
71
|
+
function placetypeToTag(placetype: WhosOnFirstPlacetype | string): ComponentTag | undefined {
|
|
72
|
+
switch (placetype) {
|
|
73
|
+
case "country":
|
|
74
|
+
case "nation":
|
|
75
|
+
return "country"
|
|
76
|
+
case "macroregion":
|
|
77
|
+
case "region":
|
|
78
|
+
return "region"
|
|
79
|
+
case "macrocounty":
|
|
80
|
+
case "county":
|
|
81
|
+
case "localadmin":
|
|
82
|
+
return "subregion"
|
|
83
|
+
case "locality":
|
|
84
|
+
return "locality"
|
|
85
|
+
case "borough":
|
|
86
|
+
case "macrohood":
|
|
87
|
+
case "neighbourhood":
|
|
88
|
+
case "microhood":
|
|
89
|
+
return "dependent_locality"
|
|
90
|
+
default:
|
|
91
|
+
return undefined
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
interface VariantSpec {
|
|
96
|
+
/** Hierarchy-variant id appended to `source_id`. */
|
|
97
|
+
suffix: string
|
|
98
|
+
|
|
99
|
+
/** Component tag → display string the adapter will hand to the runner. */
|
|
100
|
+
components: Partial<Record<ComponentTag, string>>
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Compute the hierarchy variants for a record given its ancestry chain and the chosen `selfName`.
|
|
105
|
+
*
|
|
106
|
+
* `selfName` is the surface form to use for the record's own component (locality / region / country / subregion).
|
|
107
|
+
* Callers pass the canonical `wof:name` for the `"default"` slot and a `name:*` localized value for variant slots;
|
|
108
|
+
* ancestor names always come from the ancestor's canonical `wof:name`.
|
|
109
|
+
*
|
|
110
|
+
* Country variants substitute `COUNTRY_DISPLAY_NAME` for the default slot so the OpenCage template produces the
|
|
111
|
+
* canonicalized form (`"United States of America"`), matching the legacy SQLite adapter's behavior.
|
|
112
|
+
*/
|
|
113
|
+
export function variantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): VariantSpec[] {
|
|
114
|
+
const selfTag = placetypeToTag(row.placetype)
|
|
115
|
+
|
|
116
|
+
if (!selfTag) return []
|
|
117
|
+
|
|
118
|
+
const region = ancestry.find((a) => placetypeToTag(a.placetype) === "region")
|
|
119
|
+
const country = ancestry.find((a) => placetypeToTag(a.placetype) === "country")
|
|
120
|
+
const countryDisplay = COUNTRY_DISPLAY_NAME[row.country] ?? country?.name ?? row.country
|
|
121
|
+
|
|
122
|
+
const variants: VariantSpec[] = []
|
|
123
|
+
|
|
124
|
+
switch (selfTag) {
|
|
125
|
+
case "locality":
|
|
126
|
+
case "dependent_locality": {
|
|
127
|
+
variants.push({ suffix: "self", components: { [selfTag]: selfName } })
|
|
128
|
+
|
|
129
|
+
if (region) {
|
|
130
|
+
variants.push({
|
|
131
|
+
suffix: "with-region",
|
|
132
|
+
components: { [selfTag]: selfName, region: region.name },
|
|
133
|
+
})
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (region && country) {
|
|
137
|
+
variants.push({
|
|
138
|
+
suffix: "with-region-country",
|
|
139
|
+
components: { [selfTag]: selfName, region: region.name, country: countryDisplay },
|
|
140
|
+
})
|
|
141
|
+
} else if (!region && country) {
|
|
142
|
+
variants.push({
|
|
143
|
+
suffix: "with-country",
|
|
144
|
+
components: { [selfTag]: selfName, country: countryDisplay },
|
|
145
|
+
})
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
return variants
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
case "region": {
|
|
152
|
+
variants.push({ suffix: "self", components: { region: selfName } })
|
|
153
|
+
|
|
154
|
+
if (country) {
|
|
155
|
+
variants.push({
|
|
156
|
+
suffix: "with-country",
|
|
157
|
+
components: { region: selfName, country: countryDisplay },
|
|
158
|
+
})
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return variants
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
case "country": {
|
|
165
|
+
variants.push({ suffix: "self", components: { country: selfName } })
|
|
166
|
+
|
|
167
|
+
return variants
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
case "subregion": {
|
|
171
|
+
variants.push({ suffix: "self", components: { subregion: selfName } })
|
|
172
|
+
|
|
173
|
+
return variants
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
default:
|
|
177
|
+
return []
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Build the per-record name-slot list. The canonical `"default"` slot uses the OpenCage-canonical country form when the
|
|
183
|
+
* record is itself a country (matches SQLite-adapter behavior); every other placetype's default slot uses `wof:name`
|
|
184
|
+
* verbatim.
|
|
185
|
+
*
|
|
186
|
+
* Subsequent slots come from `name:*` variants, deduplicated against the default name so we don't emit a redundant
|
|
187
|
+
* `"default"`-equivalent row under a localized key.
|
|
188
|
+
*/
|
|
189
|
+
export function nameSlotsFor(rec: WOFRecord): Array<{ key: string; value: string }> {
|
|
190
|
+
const selfTag = placetypeToTag(rec.placetype)
|
|
191
|
+
const canonicalSelfName = selfTag === "country" ? (COUNTRY_DISPLAY_NAME[rec.country] ?? rec.name) : rec.name
|
|
192
|
+
|
|
193
|
+
const seen = new Set<string>([canonicalSelfName])
|
|
194
|
+
const slots: Array<{ key: string; value: string }> = [{ key: "default", value: canonicalSelfName }]
|
|
195
|
+
|
|
196
|
+
for (const [rawKey, value] of rec.nameVariants) {
|
|
197
|
+
if (seen.has(value)) continue
|
|
198
|
+
seen.add(value)
|
|
199
|
+
slots.push({ key: normalizeNameKey(rawKey), value })
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return slots
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
export const WOF_ADMIN_ADAPTER_ID = "wof-admin"
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Construct the wof-admin JSON-bundle adapter. The adapter is stateless across runs; calling this twice with the same
|
|
209
|
+
* input directory produces byte-identical `canonical.jsonl` (records are emitted in sorted `wof:id` order to be
|
|
210
|
+
* insensitive to filesystem walk ordering).
|
|
211
|
+
*/
|
|
212
|
+
export function createWOFAdminAdapter(): CorpusAdapter {
|
|
213
|
+
return {
|
|
214
|
+
id: WOF_ADMIN_ADAPTER_ID,
|
|
215
|
+
defaultLicense: "CC0-1.0",
|
|
216
|
+
description:
|
|
217
|
+
"Who's On First admin GeoJSON bundles (countries, regions, counties, localities) — multi-name variants per record.",
|
|
218
|
+
|
|
219
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
220
|
+
// Pass 1: scan every GeoJSON file once, build the in-memory record index.
|
|
221
|
+
// We keep only records whose placetype maps to a ComponentTag — irrelevant placetypes
|
|
222
|
+
// (campus, county-region hybrids on which Mailwoman has no opinion) are dropped here so
|
|
223
|
+
// they don't inflate the ancestry index. Country-filtered runs prune to the matching
|
|
224
|
+
// country code too; the ancestors of a same-country record live in the same admin repo.
|
|
225
|
+
const byID = new Map<number, WOFRecord>()
|
|
226
|
+
|
|
227
|
+
for await (const rec of walkFeatures(opts.inputPath, { signal: opts.signal })) {
|
|
228
|
+
if (opts.signal?.aborted) return
|
|
229
|
+
|
|
230
|
+
if (opts.country && rec.country !== opts.country) continue
|
|
231
|
+
|
|
232
|
+
if (!placetypeToTag(rec.placetype)) continue
|
|
233
|
+
byID.set(rec.id, rec)
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
const ancestry = buildAncestryIndex(byID)
|
|
237
|
+
|
|
238
|
+
// Pass 2: emit rows in sorted-id order for deterministic JSONL.
|
|
239
|
+
const ids = [...byID.keys()].sort((a, b) => a - b)
|
|
240
|
+
let emitted = 0
|
|
241
|
+
|
|
242
|
+
for (const id of ids) {
|
|
243
|
+
if (opts.signal?.aborted) return
|
|
244
|
+
const rec = byID.get(id)!
|
|
245
|
+
const chain = ancestry.get(id) ?? []
|
|
246
|
+
const slots = nameSlotsFor(rec)
|
|
247
|
+
|
|
248
|
+
for (const slot of slots) {
|
|
249
|
+
const variants = variantsFor(rec, chain, slot.value)
|
|
250
|
+
|
|
251
|
+
for (const variant of variants) {
|
|
252
|
+
if (opts.limit !== undefined && emitted >= opts.limit) return
|
|
253
|
+
|
|
254
|
+
const raw = formatAddress(variant.components, rec.country, { separator: ", " })
|
|
255
|
+
|
|
256
|
+
if (!raw) continue
|
|
257
|
+
const aligned = reconcileComponents(variant.components, raw)
|
|
258
|
+
|
|
259
|
+
if (Object.keys(aligned).length === 0) continue
|
|
260
|
+
|
|
261
|
+
yield {
|
|
262
|
+
raw,
|
|
263
|
+
components: aligned,
|
|
264
|
+
country: rec.country,
|
|
265
|
+
locale: LOCALE_BY_COUNTRY[rec.country],
|
|
266
|
+
source: WOF_ADMIN_ADAPTER_ID,
|
|
267
|
+
source_id: `${WOF_ADMIN_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
|
|
268
|
+
corpus_version: "",
|
|
269
|
+
license: "CC0-1.0",
|
|
270
|
+
}
|
|
271
|
+
emitted++
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
},
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
/** Single shared instance, suitable for `defaultAdapterRegistry`. */
|
|
280
|
+
export const wofAdminAdapter = createWOFAdminAdapter()
|