@mailwoman/corpus 7.2.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +59 -27
- package/src/adapter.ts +153 -0
- package/src/adapters/ban/adapter.ts +189 -0
- package/src/adapters/ban/street-decompose.ts +98 -0
- package/src/adapters/fcc-bdc/adapter.ts +188 -0
- package/src/adapters/geonames/adapter.ts +185 -0
- package/src/adapters/geonames-postal/adapter.ts +111 -0
- package/src/adapters/gnaf/adapter.ts +139 -0
- package/src/adapters/gnaf/assemble.ts +200 -0
- package/src/adapters/index.ts +135 -0
- package/src/adapters/openaddresses/adapter.ts +237 -0
- package/src/adapters/overture/adapter.ts +149 -0
- package/src/adapters/state-hi-schools/adapter.ts +171 -0
- package/src/adapters/state-ia-contractors/adapter.ts +148 -0
- package/src/adapters/state-ny-notaries/adapter.ts +169 -0
- package/src/adapters/state-tx-notaries/adapter.ts +160 -0
- package/src/adapters/synth-po-box/adapter.ts +197 -0
- package/src/adapters/tiger/adapter.ts +232 -0
- package/src/adapters/tiger/street-decompose.ts +120 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
- package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
- package/src/adapters/usgov-nad/adapter.ts +338 -0
- package/src/adapters/usgov-nppes/adapter.ts +169 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
- package/src/adapters/wof-admin-jp/adapter.ts +176 -0
- package/src/adapters/wof-admin-json/adapter.ts +280 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
- package/src/align.ts +317 -0
- package/src/build.ts +348 -0
- package/src/codex/us-fips-state.ts +109 -0
- package/src/format.ts +14 -0
- package/src/golden.ts +147 -0
- package/src/index.ts +28 -0
- package/src/license.ts +48 -0
- package/src/parquet-wrapper/index.ts +9 -0
- package/src/parquet-wrapper/reader.ts +67 -0
- package/src/parquet-wrapper/schema.ts +80 -0
- package/src/parquet-wrapper/writer.ts +102 -0
- package/src/parquet.ts +360 -0
- package/src/runner.ts +283 -0
- package/src/shard-recipes/anchor-absorption.ts +86 -0
- package/src/shard-recipes/boundary-stress.ts +87 -0
- package/src/shard-recipes/country-balanced.ts +471 -0
- package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
- package/src/shard-recipes/fr-admin-split.ts +212 -0
- package/src/shard-recipes/fr-bare-street.ts +85 -0
- package/src/shard-recipes/fr-fragment.ts +304 -0
- package/src/shard-recipes/fr-order.ts +283 -0
- package/src/shard-recipes/german.ts +206 -0
- package/src/shard-recipes/house-venue.ts +78 -0
- package/src/shard-recipes/index.ts +76 -0
- package/src/shard-recipes/intersection.ts +617 -0
- package/src/shard-recipes/locale.ts +415 -0
- package/src/shard-recipes/nl-postcode.ts +105 -0
- package/src/shard-recipes/no-fragment.ts +206 -0
- package/src/shard-recipes/no-street-led.ts +130 -0
- package/src/shard-recipes/no-street.ts +78 -0
- package/src/shard-recipes/po-box-cedex.ts +853 -0
- package/src/shard-recipes/po-box.ts +111 -0
- package/src/shard-recipes/scaffold.ts +187 -0
- package/src/shard-recipes/si-bare-village.ts +88 -0
- package/src/shard-recipes/street-affix.ts +549 -0
- package/src/shard-recipes/street-bare.ts +90 -0
- package/src/shard-recipes/street.ts +74 -0
- package/src/shard-recipes/unit.ts +315 -0
- package/src/split.ts +251 -0
- package/src/synthesize-anchor-absorption.ts +263 -0
- package/src/synthesize-boundary-stress.ts +504 -0
- package/src/synthesize-german.ts +227 -0
- package/src/synthesize-house-venue.ts +192 -0
- package/src/synthesize-intersection.ts +205 -0
- package/src/synthesize-no-street.ts +341 -0
- package/src/synthesize-po-box.ts +307 -0
- package/src/synthesize-street.ts +264 -0
- package/src/synthesize.ts +872 -0
- package/src/tokenize.ts +68 -0
- package/src/tools/align-shard.ts +67 -0
- package/src/tools/audit.ts +352 -0
- package/src/tools/corpus-stats.ts +183 -0
- package/src/tools/fetch/ban.ts +288 -0
- package/src/tools/fetch/download.ts +119 -0
- package/src/tools/fetch/hrsa.ts +65 -0
- package/src/tools/fetch/imls-pls.ts +162 -0
- package/src/tools/fetch/index.ts +121 -0
- package/src/tools/fetch/nad.ts +304 -0
- package/src/tools/fetch/nppes.ts +177 -0
- package/src/tools/fetch/openaddresses.ts +380 -0
- package/src/tools/fetch/state-hi-schools.ts +217 -0
- package/src/tools/fetch/state-sources.ts +142 -0
- package/src/tools/fetch/tiger-full.ts +318 -0
- package/src/tools/golden-expand.ts +524 -0
- package/src/tools/golden-promote.ts +304 -0
- package/src/tools/index.ts +24 -0
- package/src/tools/ingest-csv.ts +420 -0
- package/src/tools/jsonl-to-parquet.ts +200 -0
- package/src/tools/lint-shard-vocab.ts +357 -0
- package/src/tools/lint-shard.ts +502 -0
- package/src/tools/overlay-manifest.ts +141 -0
- package/src/tools/shard-kryptonite.ts +131 -0
- package/src/tools/shard-translit.ts +244 -0
- package/src/types.ts +213 -0
- package/src/wof-json.ts +236 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
|
|
7
|
+
* Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
|
|
8
|
+
* "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
|
|
9
|
+
* reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
|
|
10
|
+
* PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
|
|
11
|
+
* (ACT/NSW/…).
|
|
12
|
+
*
|
|
13
|
+
* Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
|
|
14
|
+
* keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
|
|
15
|
+
* which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
|
|
16
|
+
* ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
|
|
17
|
+
* bounded (the OOM lesson from the Overture ingest).
|
|
18
|
+
*
|
|
19
|
+
* No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
|
|
20
|
+
* AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
|
|
21
|
+
* pinned (65%→87% if the parse were order-robust). The parser needs the address string +
|
|
22
|
+
* component labels, not lat/lon.
|
|
23
|
+
*
|
|
24
|
+
* Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
|
|
25
|
+
* multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
|
|
26
|
+
* excluded by (street, locality, postcode) so the training shard never overlaps the benchmark.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
import { createWriteStream } from "node:fs"
|
|
30
|
+
import { readdir, readFile } from "node:fs/promises"
|
|
31
|
+
import { join } from "node:path"
|
|
32
|
+
|
|
33
|
+
import { PSVSpliterator } from "spliterator"
|
|
34
|
+
|
|
35
|
+
export interface GNAFAssembleOptions {
|
|
36
|
+
/** G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables). */
|
|
37
|
+
standardDir: string
|
|
38
|
+
/** Target sample size (uniform reservoir → population-proportional across states). */
|
|
39
|
+
sampleSize: number
|
|
40
|
+
/** Output JSONL path. */
|
|
41
|
+
out: string
|
|
42
|
+
/**
|
|
43
|
+
* Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
|
|
44
|
+
*/
|
|
45
|
+
holdoutPath?: string
|
|
46
|
+
/** Progress sink (the CLI passes a setter). */
|
|
47
|
+
onProgress?: (message: string) => void
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface GNAFAssembleResult {
|
|
51
|
+
written: number
|
|
52
|
+
seen: number
|
|
53
|
+
heldOut: number
|
|
54
|
+
byState: Record<string, number>
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/** UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour). */
|
|
58
|
+
export function titlecase(s: string): string {
|
|
59
|
+
return s
|
|
60
|
+
.toLowerCase()
|
|
61
|
+
.replace(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
|
|
62
|
+
.trim()
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative). */
|
|
66
|
+
export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
|
|
67
|
+
return `${street}|${locality}|${postcode}`.toLowerCase()
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
type Row = Record<string, string | number | undefined>
|
|
71
|
+
async function* psvObjects(path: string): AsyncIterable<Row> {
|
|
72
|
+
yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/** Load a small lookup table fully into a Map keyed by `keyCol`. */
|
|
76
|
+
async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
|
|
77
|
+
const m = new Map<string, V>()
|
|
78
|
+
|
|
79
|
+
for (const p of paths) {
|
|
80
|
+
for await (const r of psvObjects(p)) {
|
|
81
|
+
const k = r[keyCol]
|
|
82
|
+
|
|
83
|
+
if (k != null && k !== "") {
|
|
84
|
+
m.set(String(k), pick(r))
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return m
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Build the held-out key set from an eval JSONL whose rows carry a `components` object. */
|
|
93
|
+
async function loadHoldout(path: string): Promise<Set<string>> {
|
|
94
|
+
const keys = new Set<string>()
|
|
95
|
+
const text = await readFile(path, "utf8")
|
|
96
|
+
|
|
97
|
+
for (const line of text.split("\n")) {
|
|
98
|
+
if (!line.trim()) continue
|
|
99
|
+
|
|
100
|
+
try {
|
|
101
|
+
const c = (JSON.parse(line) as { components?: Record<string, string> }).components
|
|
102
|
+
|
|
103
|
+
if (c?.street && c?.locality && c?.postcode) {
|
|
104
|
+
keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
|
|
105
|
+
}
|
|
106
|
+
} catch {
|
|
107
|
+
/* skip malformed */
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return keys
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
|
|
115
|
+
const progress = opts.onProgress ?? (() => {})
|
|
116
|
+
const files = await readdir(opts.standardDir)
|
|
117
|
+
const pick = (re: RegExp, exclude?: RegExp) =>
|
|
118
|
+
files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
|
|
119
|
+
|
|
120
|
+
// `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
|
|
121
|
+
const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
|
|
122
|
+
const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
|
|
123
|
+
const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
|
|
124
|
+
|
|
125
|
+
const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
|
|
126
|
+
|
|
127
|
+
if (opts.holdoutPath) {
|
|
128
|
+
progress(`held-out eval keys: ${holdout.size}`)
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
|
|
132
|
+
const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
|
|
133
|
+
name: String(r.STREET_NAME ?? ""),
|
|
134
|
+
type: String(r.STREET_TYPE_CODE ?? ""),
|
|
135
|
+
suffix: String(r.STREET_SUFFIX_CODE ?? ""),
|
|
136
|
+
}))
|
|
137
|
+
const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
|
|
138
|
+
progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
|
|
139
|
+
|
|
140
|
+
const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
|
|
141
|
+
[]
|
|
142
|
+
let seen = 0
|
|
143
|
+
let heldOut = 0
|
|
144
|
+
|
|
145
|
+
for (const p of addressPaths) {
|
|
146
|
+
const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
|
|
147
|
+
|
|
148
|
+
for await (const r of psvObjects(p)) {
|
|
149
|
+
const numberFirst = String(r.NUMBER_FIRST ?? "")
|
|
150
|
+
|
|
151
|
+
if (!numberFirst || r.DATE_RETIRED || !r.POSTCODE) continue
|
|
152
|
+
const st = streetMap.get(String(r.STREET_LOCALITY_PID ?? ""))
|
|
153
|
+
const suburbRaw = localityMap.get(String(r.LOCALITY_PID ?? ""))
|
|
154
|
+
|
|
155
|
+
if (!st?.name || !suburbRaw) continue
|
|
156
|
+
const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim()
|
|
157
|
+
const locality = titlecase(suburbRaw)
|
|
158
|
+
const postcode = String(r.POSTCODE)
|
|
159
|
+
|
|
160
|
+
if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
|
|
161
|
+
heldOut++
|
|
162
|
+
continue
|
|
163
|
+
}
|
|
164
|
+
let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
|
|
165
|
+
|
|
166
|
+
if (r.NUMBER_LAST) {
|
|
167
|
+
house = `${house}-${String(r.NUMBER_LAST)}`
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
if (r.FLAT_NUMBER) {
|
|
171
|
+
house = `${String(r.FLAT_NUMBER)}/${house}`
|
|
172
|
+
}
|
|
173
|
+
const tuple = { house_number: house, street, locality, region: state, postcode }
|
|
174
|
+
seen++
|
|
175
|
+
|
|
176
|
+
if (reservoir.length < opts.sampleSize) {
|
|
177
|
+
reservoir.push(tuple)
|
|
178
|
+
} else {
|
|
179
|
+
const j = Math.floor(Math.random() * seen)
|
|
180
|
+
|
|
181
|
+
if (j < opts.sampleSize) {
|
|
182
|
+
reservoir[j] = tuple
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const out = createWriteStream(opts.out)
|
|
190
|
+
const byState: Record<string, number> = {}
|
|
191
|
+
|
|
192
|
+
for (const t of reservoir) {
|
|
193
|
+
out.write(JSON.stringify(t) + "\n")
|
|
194
|
+
byState[t.region] = (byState[t.region] ?? 0) + 1
|
|
195
|
+
}
|
|
196
|
+
await new Promise<void>((res) => out.end(res))
|
|
197
|
+
progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
|
|
198
|
+
|
|
199
|
+
return { written: reservoir.length, seen, heldOut, byState }
|
|
200
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Adapter registry bootstrap.
|
|
7
|
+
*
|
|
8
|
+
* Importing this module registers every built-in adapter with `defaultAdapterRegistry`. The CLI
|
|
9
|
+
* (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
|
|
10
|
+
*
|
|
11
|
+
* Adapters under construction live in their own subdirectories (`./wof-admin-json/`, `./ban/`, ...)
|
|
12
|
+
* and are added to the `BUILTIN_ADAPTERS` list here as they come online. Tests that need a
|
|
13
|
+
* pristine registry should construct their own `InMemoryAdapterRegistry` instead of mutating the
|
|
14
|
+
* default.
|
|
15
|
+
*
|
|
16
|
+
* The WOF adapters export their canonical ids — `wof-admin` and `wof-postalcode` — so existing
|
|
17
|
+
* `mailwoman corpus build` callsites do not need to change despite the Phase 1.5.1 SQLite →
|
|
18
|
+
* JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
|
|
19
|
+
* implementations; the registered ids are unchanged).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { defaultAdapterRegistry } from "../adapter.ts"
|
|
23
|
+
import type { CorpusAdapter } from "../types.ts"
|
|
24
|
+
import { banAdapter } from "./ban/adapter.ts"
|
|
25
|
+
import { fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
26
|
+
import { geonamesPostalAdapter } from "./geonames-postal/adapter.ts"
|
|
27
|
+
import { geonamesAdapter } from "./geonames/adapter.ts"
|
|
28
|
+
import { gnafAdapter } from "./gnaf/adapter.ts"
|
|
29
|
+
import { openaddressesAdapter } from "./openaddresses/adapter.ts"
|
|
30
|
+
import { overtureAdapter } from "./overture/adapter.ts"
|
|
31
|
+
import { stateHiSchoolsAdapter } from "./state-hi-schools/adapter.ts"
|
|
32
|
+
import { stateIaContractorsAdapter } from "./state-ia-contractors/adapter.ts"
|
|
33
|
+
import { stateNyNotariesAdapter } from "./state-ny-notaries/adapter.ts"
|
|
34
|
+
import { stateTxNotariesAdapter } from "./state-tx-notaries/adapter.ts"
|
|
35
|
+
import { tigerAdapter } from "./tiger/adapter.ts"
|
|
36
|
+
import { usgovHrsaFqhcAdapter } from "./usgov-hrsa-fqhc/adapter.ts"
|
|
37
|
+
import { usgovImlsPlsAdapter } from "./usgov-imls-pls/adapter.ts"
|
|
38
|
+
import { usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
39
|
+
import { usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
40
|
+
import { usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
41
|
+
import { wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
42
|
+
import { wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
|
|
46
|
+
* Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
|
|
47
|
+
* (FCC BDC US, HRSA FQHC US).
|
|
48
|
+
*
|
|
49
|
+
* The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list — the SAMHSA Open Data Foundry
|
|
50
|
+
* bulk CSV the adapter was written against is no longer publicly distributed (see issue #33, 2026-05-17 investigation).
|
|
51
|
+
* The factory + named export remain available so the adapter can be hand-registered if an operator obtains a compatible
|
|
52
|
+
* CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
|
|
53
|
+
*/
|
|
54
|
+
export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
|
|
55
|
+
wofAdminAdapter,
|
|
56
|
+
wofPostalcodeAdapter,
|
|
57
|
+
geonamesAdapter,
|
|
58
|
+
geonamesPostalAdapter,
|
|
59
|
+
banAdapter,
|
|
60
|
+
tigerAdapter,
|
|
61
|
+
openaddressesAdapter,
|
|
62
|
+
overtureAdapter,
|
|
63
|
+
gnafAdapter,
|
|
64
|
+
fccBdcAdapter,
|
|
65
|
+
usgovHrsaFqhcAdapter,
|
|
66
|
+
usgovNPPESAdapter,
|
|
67
|
+
usgovNADAdapter,
|
|
68
|
+
usgovImlsPlsAdapter,
|
|
69
|
+
usgovIrsBmfAdapter,
|
|
70
|
+
stateIaContractorsAdapter,
|
|
71
|
+
stateTxNotariesAdapter,
|
|
72
|
+
stateNyNotariesAdapter,
|
|
73
|
+
stateHiSchoolsAdapter,
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
for (const adapter of BUILTIN_ADAPTERS) {
|
|
77
|
+
if (!defaultAdapterRegistry.get(adapter.id)) {
|
|
78
|
+
defaultAdapterRegistry.register(adapter)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
|
|
83
|
+
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
84
|
+
export {
|
|
85
|
+
GEONAMES_POSTAL_ADAPTER_ID,
|
|
86
|
+
GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
87
|
+
geonamesPostalAdapter,
|
|
88
|
+
} from "./geonames-postal/adapter.ts"
|
|
89
|
+
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
|
|
90
|
+
export {
|
|
91
|
+
OPENADDRESSES_ADAPTER_ID,
|
|
92
|
+
OPENADDRESSES_DEFAULT_LICENSE,
|
|
93
|
+
openaddressesAdapter,
|
|
94
|
+
} from "./openaddresses/adapter.ts"
|
|
95
|
+
export {
|
|
96
|
+
STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
97
|
+
STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
98
|
+
stateHiSchoolsAdapter,
|
|
99
|
+
} from "./state-hi-schools/adapter.ts"
|
|
100
|
+
export {
|
|
101
|
+
STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
102
|
+
STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
103
|
+
stateIaContractorsAdapter,
|
|
104
|
+
} from "./state-ia-contractors/adapter.ts"
|
|
105
|
+
export {
|
|
106
|
+
STATE_NY_NOTARIES_ADAPTER_ID,
|
|
107
|
+
STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
108
|
+
stateNyNotariesAdapter,
|
|
109
|
+
} from "./state-ny-notaries/adapter.ts"
|
|
110
|
+
export {
|
|
111
|
+
STATE_TX_NOTARIES_ADAPTER_ID,
|
|
112
|
+
STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
113
|
+
stateTxNotariesAdapter,
|
|
114
|
+
} from "./state-tx-notaries/adapter.ts"
|
|
115
|
+
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
|
|
116
|
+
export {
|
|
117
|
+
USGOV_HRSA_FQHC_ADAPTER_ID,
|
|
118
|
+
USGOV_HRSA_FQHC_DEFAULT_LICENSE,
|
|
119
|
+
usgovHrsaFqhcAdapter,
|
|
120
|
+
} from "./usgov-hrsa-fqhc/adapter.ts"
|
|
121
|
+
export {
|
|
122
|
+
USGOV_IMLS_PLS_ADAPTER_ID,
|
|
123
|
+
USGOV_IMLS_PLS_DEFAULT_LICENSE,
|
|
124
|
+
usgovImlsPlsAdapter,
|
|
125
|
+
} from "./usgov-imls-pls/adapter.ts"
|
|
126
|
+
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
127
|
+
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
128
|
+
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
129
|
+
export {
|
|
130
|
+
USGOV_SAMHSA_ADAPTER_ID,
|
|
131
|
+
USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
132
|
+
usgovSamhsaTreatmentLocatorAdapter,
|
|
133
|
+
} from "./usgov-samhsa-treatment-locator/adapter.ts"
|
|
134
|
+
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
135
|
+
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
|
|
7
|
+
*
|
|
8
|
+
* OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
|
|
9
|
+
* (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
|
|
10
|
+
* line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
|
|
11
|
+
* matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
|
|
12
|
+
*
|
|
13
|
+
* The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
|
|
14
|
+
* open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
|
|
15
|
+
* per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
|
|
16
|
+
* propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
|
|
17
|
+
* or re-attribute by license at training time.
|
|
18
|
+
*
|
|
19
|
+
* Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
|
|
20
|
+
* but the row-level data doesn't include a country code, so the adapter refuses to run without
|
|
21
|
+
* one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
|
|
22
|
+
* inputs JSON.
|
|
23
|
+
*
|
|
24
|
+
* Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
|
|
25
|
+
* variants are accepted because legacy dumps used UPPERCASE):
|
|
26
|
+
*
|
|
27
|
+
* | Property | ComponentTag | | ------------- |
|
|
28
|
+
* -------------------------------------------------------------- | | `number` | `house_number` |
|
|
29
|
+
* | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
|
|
30
|
+
* `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
|
|
31
|
+
* per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
|
|
32
|
+
* then synthesize)|
|
|
33
|
+
*
|
|
34
|
+
* `district` is intentionally NOT mapped — for US data it carries borough or county and would
|
|
35
|
+
* inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
|
|
36
|
+
* for non-US locales where district names DO appear on the envelope.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { TextSpliterator } from "spliterator"
|
|
40
|
+
|
|
41
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
42
|
+
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
43
|
+
import { SHARE_ALIKE_PATTERN } from "../../license.ts"
|
|
44
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
45
|
+
|
|
46
|
+
export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
|
|
47
|
+
export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
|
|
51
|
+
* this interface documents the canonical lowercase form after normalization.
|
|
52
|
+
*/
|
|
53
|
+
interface OaProperties {
|
|
54
|
+
hash?: string
|
|
55
|
+
id?: string
|
|
56
|
+
number?: string
|
|
57
|
+
street?: string
|
|
58
|
+
unit?: string
|
|
59
|
+
city?: string
|
|
60
|
+
district?: string
|
|
61
|
+
region?: string
|
|
62
|
+
postcode?: string
|
|
63
|
+
license?: string
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
|
|
67
|
+
function normalizeProperties(raw: unknown): OaProperties {
|
|
68
|
+
if (!raw || typeof raw !== "object") return {}
|
|
69
|
+
const out: Record<string, string> = {}
|
|
70
|
+
|
|
71
|
+
for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
|
|
72
|
+
if (typeof v === "string") {
|
|
73
|
+
out[k.toLowerCase()] = v
|
|
74
|
+
} else if (typeof v === "number") {
|
|
75
|
+
out[k.toLowerCase()] = String(v)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return out as OaProperties
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes. */
|
|
83
|
+
function parseFeatureLine(line: string): OaProperties | null {
|
|
84
|
+
const trimmed = line.trim()
|
|
85
|
+
|
|
86
|
+
if (!trimmed || trimmed.startsWith("#")) return null
|
|
87
|
+
let parsed: unknown
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(trimmed)
|
|
91
|
+
} catch {
|
|
92
|
+
return null
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (!parsed || typeof parsed !== "object") return null
|
|
96
|
+
const obj = parsed as { type?: string; properties?: unknown }
|
|
97
|
+
|
|
98
|
+
if (obj.type !== "Feature") return null
|
|
99
|
+
|
|
100
|
+
return normalizeProperties(obj.properties)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface OpenaddressesAdapterOptions {
|
|
104
|
+
/**
|
|
105
|
+
* Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
|
|
106
|
+
* license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
|
|
107
|
+
*/
|
|
108
|
+
defaultLicense?: string
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
|
|
112
|
+
* act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
|
|
113
|
+
* exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
|
|
114
|
+
* `--exclude-share-alike` is the normal path.
|
|
115
|
+
*/
|
|
116
|
+
allowShareAlike?: boolean
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
|
|
121
|
+
* known to carry a single license throughout (e.g. a PDDL-only state slice).
|
|
122
|
+
*/
|
|
123
|
+
export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
|
|
124
|
+
const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
|
|
125
|
+
const allowShareAlike = opts.allowShareAlike ?? true
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
id: OPENADDRESSES_ADAPTER_ID,
|
|
129
|
+
defaultLicense,
|
|
130
|
+
description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
|
|
131
|
+
|
|
132
|
+
async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
133
|
+
if (!adapterOpts.country) {
|
|
134
|
+
throw new Error(
|
|
135
|
+
"openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
|
|
136
|
+
)
|
|
137
|
+
}
|
|
138
|
+
const country = adapterOpts.country
|
|
139
|
+
|
|
140
|
+
// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
|
|
141
|
+
// malformed lines by returning null); passing the path string lets the lib own + dispose
|
|
142
|
+
// the file handle, including on an early `break`.
|
|
143
|
+
const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
|
|
144
|
+
|
|
145
|
+
let emitted = 0
|
|
146
|
+
let shareAlikeBlocked = 0
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
for await (const line of lines) {
|
|
150
|
+
if (adapterOpts.signal?.aborted) break
|
|
151
|
+
|
|
152
|
+
if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
|
|
153
|
+
|
|
154
|
+
const props = parseFeatureLine(line)
|
|
155
|
+
|
|
156
|
+
if (!props) continue
|
|
157
|
+
|
|
158
|
+
const houseNumber = props.number?.trim() ?? ""
|
|
159
|
+
const street = props.street?.trim() ?? ""
|
|
160
|
+
const unit = props.unit?.trim() ?? ""
|
|
161
|
+
const city = props.city?.trim() ?? ""
|
|
162
|
+
const region = props.region?.trim() ?? ""
|
|
163
|
+
const postcode = props.postcode?.trim() ?? ""
|
|
164
|
+
|
|
165
|
+
// A row is only useful if it has, at minimum, a street + (postcode OR locality).
|
|
166
|
+
// Pure point-only rows would land in quarantine anyway.
|
|
167
|
+
if (!street) continue
|
|
168
|
+
|
|
169
|
+
if (!city && !postcode) continue
|
|
170
|
+
|
|
171
|
+
const license = (props.license?.trim() || defaultLicense).trim()
|
|
172
|
+
|
|
173
|
+
if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
|
|
174
|
+
shareAlikeBlocked++
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const components: CanonicalRow["components"] = {}
|
|
179
|
+
|
|
180
|
+
if (houseNumber) {
|
|
181
|
+
components.house_number = houseNumber
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (street) {
|
|
185
|
+
components.street = street
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (unit) {
|
|
189
|
+
components.unit = unit
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (city) {
|
|
193
|
+
components.locality = city
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (region) {
|
|
197
|
+
components.region = region
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (postcode) {
|
|
201
|
+
components.postcode = postcode
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const raw = formatAddress(components, country, { separator: ", " })
|
|
205
|
+
|
|
206
|
+
if (!raw) continue
|
|
207
|
+
|
|
208
|
+
const aligned = reconcileComponents(components, raw)
|
|
209
|
+
|
|
210
|
+
if (Object.keys(aligned).length === 0) continue
|
|
211
|
+
|
|
212
|
+
const sourceIDSeed = props.hash?.trim() || props.id?.trim()
|
|
213
|
+
const sourceID = sourceIDSeed
|
|
214
|
+
? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
|
|
215
|
+
: stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
|
|
216
|
+
|
|
217
|
+
yield {
|
|
218
|
+
raw,
|
|
219
|
+
components: aligned,
|
|
220
|
+
country,
|
|
221
|
+
source: OPENADDRESSES_ADAPTER_ID,
|
|
222
|
+
source_id: sourceID,
|
|
223
|
+
corpus_version: "",
|
|
224
|
+
license,
|
|
225
|
+
}
|
|
226
|
+
emitted++
|
|
227
|
+
}
|
|
228
|
+
} finally {
|
|
229
|
+
if (shareAlikeBlocked > 0) {
|
|
230
|
+
process.stderr.write(` openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
},
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export const openaddressesAdapter = createOpenaddressesAdapter()
|