@mailwoman/corpus 7.1.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +15 -7
- package/src/adapter.ts +153 -0
- package/src/adapters/ban/adapter.ts +189 -0
- package/src/adapters/ban/street-decompose.ts +98 -0
- package/src/adapters/fcc-bdc/adapter.ts +188 -0
- package/src/adapters/geonames/adapter.ts +185 -0
- package/src/adapters/geonames-postal/adapter.ts +111 -0
- package/src/adapters/gnaf/adapter.ts +139 -0
- package/src/adapters/gnaf/assemble.ts +200 -0
- package/src/adapters/index.ts +135 -0
- package/src/adapters/openaddresses/adapter.ts +237 -0
- package/src/adapters/overture/adapter.ts +149 -0
- package/src/adapters/state-hi-schools/adapter.ts +171 -0
- package/src/adapters/state-ia-contractors/adapter.ts +148 -0
- package/src/adapters/state-ny-notaries/adapter.ts +169 -0
- package/src/adapters/state-tx-notaries/adapter.ts +160 -0
- package/src/adapters/synth-po-box/adapter.ts +197 -0
- package/src/adapters/tiger/adapter.ts +232 -0
- package/src/adapters/tiger/street-decompose.ts +120 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
- package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
- package/src/adapters/usgov-nad/adapter.ts +338 -0
- package/src/adapters/usgov-nppes/adapter.ts +169 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
- package/src/adapters/wof-admin-jp/adapter.ts +176 -0
- package/src/adapters/wof-admin-json/adapter.ts +280 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
- package/src/align.ts +317 -0
- package/src/build.ts +348 -0
- package/src/codex/us-fips-state.ts +109 -0
- package/src/format.ts +14 -0
- package/src/golden.ts +147 -0
- package/src/index.ts +28 -0
- package/src/license.ts +48 -0
- package/src/parquet-wrapper/index.ts +9 -0
- package/src/parquet-wrapper/reader.ts +67 -0
- package/src/parquet-wrapper/schema.ts +80 -0
- package/src/parquet-wrapper/writer.ts +102 -0
- package/src/parquet.ts +360 -0
- package/src/runner.ts +283 -0
- package/src/shard-recipes/anchor-absorption.ts +86 -0
- package/src/shard-recipes/boundary-stress.ts +87 -0
- package/src/shard-recipes/country-balanced.ts +471 -0
- package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
- package/src/shard-recipes/fr-admin-split.ts +212 -0
- package/src/shard-recipes/fr-bare-street.ts +85 -0
- package/src/shard-recipes/fr-fragment.ts +304 -0
- package/src/shard-recipes/fr-order.ts +283 -0
- package/src/shard-recipes/german.ts +206 -0
- package/src/shard-recipes/house-venue.ts +78 -0
- package/src/shard-recipes/index.ts +76 -0
- package/src/shard-recipes/intersection.ts +617 -0
- package/src/shard-recipes/locale.ts +415 -0
- package/src/shard-recipes/nl-postcode.ts +105 -0
- package/src/shard-recipes/no-fragment.ts +206 -0
- package/src/shard-recipes/no-street-led.ts +130 -0
- package/src/shard-recipes/no-street.ts +78 -0
- package/src/shard-recipes/po-box-cedex.ts +853 -0
- package/src/shard-recipes/po-box.ts +111 -0
- package/src/shard-recipes/scaffold.ts +187 -0
- package/src/shard-recipes/si-bare-village.ts +88 -0
- package/src/shard-recipes/street-affix.ts +549 -0
- package/src/shard-recipes/street-bare.ts +90 -0
- package/src/shard-recipes/street.ts +74 -0
- package/src/shard-recipes/unit.ts +315 -0
- package/src/split.ts +251 -0
- package/src/synthesize-anchor-absorption.ts +263 -0
- package/src/synthesize-boundary-stress.ts +504 -0
- package/src/synthesize-german.ts +227 -0
- package/src/synthesize-house-venue.ts +192 -0
- package/src/synthesize-intersection.ts +205 -0
- package/src/synthesize-no-street.ts +341 -0
- package/src/synthesize-po-box.ts +307 -0
- package/src/synthesize-street.ts +264 -0
- package/src/synthesize.ts +872 -0
- package/src/tokenize.ts +68 -0
- package/src/tools/align-shard.ts +67 -0
- package/src/tools/audit.ts +352 -0
- package/src/tools/corpus-stats.ts +183 -0
- package/src/tools/fetch/ban.ts +288 -0
- package/src/tools/fetch/download.ts +119 -0
- package/src/tools/fetch/hrsa.ts +65 -0
- package/src/tools/fetch/imls-pls.ts +162 -0
- package/src/tools/fetch/index.ts +121 -0
- package/src/tools/fetch/nad.ts +304 -0
- package/src/tools/fetch/nppes.ts +177 -0
- package/src/tools/fetch/openaddresses.ts +380 -0
- package/src/tools/fetch/state-hi-schools.ts +217 -0
- package/src/tools/fetch/state-sources.ts +142 -0
- package/src/tools/fetch/tiger-full.ts +318 -0
- package/src/tools/golden-expand.ts +524 -0
- package/src/tools/golden-promote.ts +304 -0
- package/src/tools/index.ts +24 -0
- package/src/tools/ingest-csv.ts +420 -0
- package/src/tools/jsonl-to-parquet.ts +200 -0
- package/src/tools/lint-shard-vocab.ts +357 -0
- package/src/tools/lint-shard.ts +502 -0
- package/src/tools/overlay-manifest.ts +141 -0
- package/src/tools/shard-kryptonite.ts +131 -0
- package/src/tools/shard-translit.ts +244 -0
- package/src/types.ts +213 -0
- package/src/wof-json.ts +236 -0
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Adapter registry bootstrap.
|
|
7
|
+
*
|
|
8
|
+
* Importing this module registers every built-in adapter with `defaultAdapterRegistry`. The CLI
|
|
9
|
+
* (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
|
|
10
|
+
*
|
|
11
|
+
* Adapters under construction live in their own subdirectories (`./wof-admin-json/`, `./ban/`, ...)
|
|
12
|
+
* and are added to the `BUILTIN_ADAPTERS` list here as they come online. Tests that need a
|
|
13
|
+
* pristine registry should construct their own `InMemoryAdapterRegistry` instead of mutating the
|
|
14
|
+
* default.
|
|
15
|
+
*
|
|
16
|
+
* The WOF adapters export their canonical ids — `wof-admin` and `wof-postalcode` — so existing
|
|
17
|
+
* `mailwoman corpus build` callsites do not need to change despite the Phase 1.5.1 SQLite →
|
|
18
|
+
* JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
|
|
19
|
+
* implementations; the registered ids are unchanged).
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { defaultAdapterRegistry } from "../adapter.ts"
|
|
23
|
+
import type { CorpusAdapter } from "../types.ts"
|
|
24
|
+
import { banAdapter } from "./ban/adapter.ts"
|
|
25
|
+
import { fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
26
|
+
import { geonamesPostalAdapter } from "./geonames-postal/adapter.ts"
|
|
27
|
+
import { geonamesAdapter } from "./geonames/adapter.ts"
|
|
28
|
+
import { gnafAdapter } from "./gnaf/adapter.ts"
|
|
29
|
+
import { openaddressesAdapter } from "./openaddresses/adapter.ts"
|
|
30
|
+
import { overtureAdapter } from "./overture/adapter.ts"
|
|
31
|
+
import { stateHiSchoolsAdapter } from "./state-hi-schools/adapter.ts"
|
|
32
|
+
import { stateIaContractorsAdapter } from "./state-ia-contractors/adapter.ts"
|
|
33
|
+
import { stateNyNotariesAdapter } from "./state-ny-notaries/adapter.ts"
|
|
34
|
+
import { stateTxNotariesAdapter } from "./state-tx-notaries/adapter.ts"
|
|
35
|
+
import { tigerAdapter } from "./tiger/adapter.ts"
|
|
36
|
+
import { usgovHrsaFqhcAdapter } from "./usgov-hrsa-fqhc/adapter.ts"
|
|
37
|
+
import { usgovImlsPlsAdapter } from "./usgov-imls-pls/adapter.ts"
|
|
38
|
+
import { usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
39
|
+
import { usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
40
|
+
import { usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
41
|
+
import { wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
42
|
+
import { wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
|
|
46
|
+
* Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
|
|
47
|
+
* (FCC BDC US, HRSA FQHC US).
|
|
48
|
+
*
|
|
49
|
+
* The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list — the SAMHSA Open Data Foundry
|
|
50
|
+
* bulk CSV the adapter was written against is no longer publicly distributed (see issue #33, 2026-05-17 investigation).
|
|
51
|
+
* The factory + named export remain available so the adapter can be hand-registered if an operator obtains a compatible
|
|
52
|
+
* CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
|
|
53
|
+
*/
|
|
54
|
+
export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
|
|
55
|
+
wofAdminAdapter,
|
|
56
|
+
wofPostalcodeAdapter,
|
|
57
|
+
geonamesAdapter,
|
|
58
|
+
geonamesPostalAdapter,
|
|
59
|
+
banAdapter,
|
|
60
|
+
tigerAdapter,
|
|
61
|
+
openaddressesAdapter,
|
|
62
|
+
overtureAdapter,
|
|
63
|
+
gnafAdapter,
|
|
64
|
+
fccBdcAdapter,
|
|
65
|
+
usgovHrsaFqhcAdapter,
|
|
66
|
+
usgovNPPESAdapter,
|
|
67
|
+
usgovNADAdapter,
|
|
68
|
+
usgovImlsPlsAdapter,
|
|
69
|
+
usgovIrsBmfAdapter,
|
|
70
|
+
stateIaContractorsAdapter,
|
|
71
|
+
stateTxNotariesAdapter,
|
|
72
|
+
stateNyNotariesAdapter,
|
|
73
|
+
stateHiSchoolsAdapter,
|
|
74
|
+
]
|
|
75
|
+
|
|
76
|
+
for (const adapter of BUILTIN_ADAPTERS) {
|
|
77
|
+
if (!defaultAdapterRegistry.get(adapter.id)) {
|
|
78
|
+
defaultAdapterRegistry.register(adapter)
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
|
|
83
|
+
export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
|
|
84
|
+
export {
|
|
85
|
+
GEONAMES_POSTAL_ADAPTER_ID,
|
|
86
|
+
GEONAMES_POSTAL_DEFAULT_LICENSE,
|
|
87
|
+
geonamesPostalAdapter,
|
|
88
|
+
} from "./geonames-postal/adapter.ts"
|
|
89
|
+
export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
|
|
90
|
+
export {
|
|
91
|
+
OPENADDRESSES_ADAPTER_ID,
|
|
92
|
+
OPENADDRESSES_DEFAULT_LICENSE,
|
|
93
|
+
openaddressesAdapter,
|
|
94
|
+
} from "./openaddresses/adapter.ts"
|
|
95
|
+
export {
|
|
96
|
+
STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
97
|
+
STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
98
|
+
stateHiSchoolsAdapter,
|
|
99
|
+
} from "./state-hi-schools/adapter.ts"
|
|
100
|
+
export {
|
|
101
|
+
STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
102
|
+
STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
103
|
+
stateIaContractorsAdapter,
|
|
104
|
+
} from "./state-ia-contractors/adapter.ts"
|
|
105
|
+
export {
|
|
106
|
+
STATE_NY_NOTARIES_ADAPTER_ID,
|
|
107
|
+
STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
108
|
+
stateNyNotariesAdapter,
|
|
109
|
+
} from "./state-ny-notaries/adapter.ts"
|
|
110
|
+
export {
|
|
111
|
+
STATE_TX_NOTARIES_ADAPTER_ID,
|
|
112
|
+
STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
113
|
+
stateTxNotariesAdapter,
|
|
114
|
+
} from "./state-tx-notaries/adapter.ts"
|
|
115
|
+
export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
|
|
116
|
+
export {
|
|
117
|
+
USGOV_HRSA_FQHC_ADAPTER_ID,
|
|
118
|
+
USGOV_HRSA_FQHC_DEFAULT_LICENSE,
|
|
119
|
+
usgovHrsaFqhcAdapter,
|
|
120
|
+
} from "./usgov-hrsa-fqhc/adapter.ts"
|
|
121
|
+
export {
|
|
122
|
+
USGOV_IMLS_PLS_ADAPTER_ID,
|
|
123
|
+
USGOV_IMLS_PLS_DEFAULT_LICENSE,
|
|
124
|
+
usgovImlsPlsAdapter,
|
|
125
|
+
} from "./usgov-imls-pls/adapter.ts"
|
|
126
|
+
export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
|
|
127
|
+
export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
|
|
128
|
+
export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
|
|
129
|
+
export {
|
|
130
|
+
USGOV_SAMHSA_ADAPTER_ID,
|
|
131
|
+
USGOV_SAMHSA_DEFAULT_LICENSE,
|
|
132
|
+
usgovSamhsaTreatmentLocatorAdapter,
|
|
133
|
+
} from "./usgov-samhsa-treatment-locator/adapter.ts"
|
|
134
|
+
export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
|
|
135
|
+
export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
|
|
7
|
+
*
|
|
8
|
+
* OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
|
|
9
|
+
* (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
|
|
10
|
+
* line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
|
|
11
|
+
* matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
|
|
12
|
+
*
|
|
13
|
+
* The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
|
|
14
|
+
* open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
|
|
15
|
+
* per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
|
|
16
|
+
* propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
|
|
17
|
+
* or re-attribute by license at training time.
|
|
18
|
+
*
|
|
19
|
+
* Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
|
|
20
|
+
* but the row-level data doesn't include a country code, so the adapter refuses to run without
|
|
21
|
+
* one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
|
|
22
|
+
* inputs JSON.
|
|
23
|
+
*
|
|
24
|
+
* Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
|
|
25
|
+
* variants are accepted because legacy dumps used UPPERCASE):
|
|
26
|
+
*
|
|
27
|
+
* | Property | ComponentTag | | ------------- |
|
|
28
|
+
* -------------------------------------------------------------- | | `number` | `house_number` |
|
|
29
|
+
* | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
|
|
30
|
+
* `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
|
|
31
|
+
* per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
|
|
32
|
+
* then synthesize)|
|
|
33
|
+
*
|
|
34
|
+
* `district` is intentionally NOT mapped — for US data it carries borough or county and would
|
|
35
|
+
* inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
|
|
36
|
+
* for non-US locales where district names DO appear on the envelope.
|
|
37
|
+
*/
|
|
38
|
+
|
|
39
|
+
import { TextSpliterator } from "spliterator"
|
|
40
|
+
|
|
41
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
42
|
+
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
43
|
+
import { SHARE_ALIKE_PATTERN } from "../../license.ts"
|
|
44
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
45
|
+
|
|
46
|
+
export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
|
|
47
|
+
export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
|
|
51
|
+
* this interface documents the canonical lowercase form after normalization.
|
|
52
|
+
*/
|
|
53
|
+
interface OaProperties {
|
|
54
|
+
hash?: string
|
|
55
|
+
id?: string
|
|
56
|
+
number?: string
|
|
57
|
+
street?: string
|
|
58
|
+
unit?: string
|
|
59
|
+
city?: string
|
|
60
|
+
district?: string
|
|
61
|
+
region?: string
|
|
62
|
+
postcode?: string
|
|
63
|
+
license?: string
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
|
|
67
|
+
function normalizeProperties(raw: unknown): OaProperties {
|
|
68
|
+
if (!raw || typeof raw !== "object") return {}
|
|
69
|
+
const out: Record<string, string> = {}
|
|
70
|
+
|
|
71
|
+
for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
|
|
72
|
+
if (typeof v === "string") {
|
|
73
|
+
out[k.toLowerCase()] = v
|
|
74
|
+
} else if (typeof v === "number") {
|
|
75
|
+
out[k.toLowerCase()] = String(v)
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
return out as OaProperties
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
/** Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes. */
|
|
83
|
+
function parseFeatureLine(line: string): OaProperties | null {
|
|
84
|
+
const trimmed = line.trim()
|
|
85
|
+
|
|
86
|
+
if (!trimmed || trimmed.startsWith("#")) return null
|
|
87
|
+
let parsed: unknown
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
parsed = JSON.parse(trimmed)
|
|
91
|
+
} catch {
|
|
92
|
+
return null
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (!parsed || typeof parsed !== "object") return null
|
|
96
|
+
const obj = parsed as { type?: string; properties?: unknown }
|
|
97
|
+
|
|
98
|
+
if (obj.type !== "Feature") return null
|
|
99
|
+
|
|
100
|
+
return normalizeProperties(obj.properties)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
export interface OpenaddressesAdapterOptions {
|
|
104
|
+
/**
|
|
105
|
+
* Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
|
|
106
|
+
* license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
|
|
107
|
+
*/
|
|
108
|
+
defaultLicense?: string
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
|
|
112
|
+
* act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
|
|
113
|
+
* exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
|
|
114
|
+
* `--exclude-share-alike` is the normal path.
|
|
115
|
+
*/
|
|
116
|
+
allowShareAlike?: boolean
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
|
|
121
|
+
* known to carry a single license throughout (e.g. a PDDL-only state slice).
|
|
122
|
+
*/
|
|
123
|
+
export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
|
|
124
|
+
const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
|
|
125
|
+
const allowShareAlike = opts.allowShareAlike ?? true
|
|
126
|
+
|
|
127
|
+
return {
|
|
128
|
+
id: OPENADDRESSES_ADAPTER_ID,
|
|
129
|
+
defaultLicense,
|
|
130
|
+
description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
|
|
131
|
+
|
|
132
|
+
async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
133
|
+
if (!adapterOpts.country) {
|
|
134
|
+
throw new Error(
|
|
135
|
+
"openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
|
|
136
|
+
)
|
|
137
|
+
}
|
|
138
|
+
const country = adapterOpts.country
|
|
139
|
+
|
|
140
|
+
// TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
|
|
141
|
+
// malformed lines by returning null); passing the path string lets the lib own + dispose
|
|
142
|
+
// the file handle, including on an early `break`.
|
|
143
|
+
const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
|
|
144
|
+
|
|
145
|
+
let emitted = 0
|
|
146
|
+
let shareAlikeBlocked = 0
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
for await (const line of lines) {
|
|
150
|
+
if (adapterOpts.signal?.aborted) break
|
|
151
|
+
|
|
152
|
+
if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
|
|
153
|
+
|
|
154
|
+
const props = parseFeatureLine(line)
|
|
155
|
+
|
|
156
|
+
if (!props) continue
|
|
157
|
+
|
|
158
|
+
const houseNumber = props.number?.trim() ?? ""
|
|
159
|
+
const street = props.street?.trim() ?? ""
|
|
160
|
+
const unit = props.unit?.trim() ?? ""
|
|
161
|
+
const city = props.city?.trim() ?? ""
|
|
162
|
+
const region = props.region?.trim() ?? ""
|
|
163
|
+
const postcode = props.postcode?.trim() ?? ""
|
|
164
|
+
|
|
165
|
+
// A row is only useful if it has, at minimum, a street + (postcode OR locality).
|
|
166
|
+
// Pure point-only rows would land in quarantine anyway.
|
|
167
|
+
if (!street) continue
|
|
168
|
+
|
|
169
|
+
if (!city && !postcode) continue
|
|
170
|
+
|
|
171
|
+
const license = (props.license?.trim() || defaultLicense).trim()
|
|
172
|
+
|
|
173
|
+
if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
|
|
174
|
+
shareAlikeBlocked++
|
|
175
|
+
continue
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const components: CanonicalRow["components"] = {}
|
|
179
|
+
|
|
180
|
+
if (houseNumber) {
|
|
181
|
+
components.house_number = houseNumber
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
if (street) {
|
|
185
|
+
components.street = street
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
if (unit) {
|
|
189
|
+
components.unit = unit
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (city) {
|
|
193
|
+
components.locality = city
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (region) {
|
|
197
|
+
components.region = region
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (postcode) {
|
|
201
|
+
components.postcode = postcode
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const raw = formatAddress(components, country, { separator: ", " })
|
|
205
|
+
|
|
206
|
+
if (!raw) continue
|
|
207
|
+
|
|
208
|
+
const aligned = reconcileComponents(components, raw)
|
|
209
|
+
|
|
210
|
+
if (Object.keys(aligned).length === 0) continue
|
|
211
|
+
|
|
212
|
+
const sourceIDSeed = props.hash?.trim() || props.id?.trim()
|
|
213
|
+
const sourceID = sourceIDSeed
|
|
214
|
+
? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
|
|
215
|
+
: stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
|
|
216
|
+
|
|
217
|
+
yield {
|
|
218
|
+
raw,
|
|
219
|
+
components: aligned,
|
|
220
|
+
country,
|
|
221
|
+
source: OPENADDRESSES_ADAPTER_ID,
|
|
222
|
+
source_id: sourceID,
|
|
223
|
+
corpus_version: "",
|
|
224
|
+
license,
|
|
225
|
+
}
|
|
226
|
+
emitted++
|
|
227
|
+
}
|
|
228
|
+
} finally {
|
|
229
|
+
if (shareAlikeBlocked > 0) {
|
|
230
|
+
process.stderr.write(` openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
},
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
export const openaddressesAdapter = createOpenaddressesAdapter()
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `overture`: Overture Maps Addresses adapter (epic #470 — the gated corpus adapter, realized
|
|
7
|
+
* 2026-06-20). Overture's global Addresses theme is the single-schema, well-normalized address
|
|
8
|
+
* dataset that fixes OpenAddresses' per-country patchiness (OA dropped Spain; OA-DE omits the
|
|
9
|
+
* Bundesland) — it even re-hosts the OA Spain data the standalone OA bucket no longer serves.
|
|
10
|
+
*
|
|
11
|
+
* This adapter consumes a per-country LINE-DELIMITED JSON dump of the corpus-relevant fields (`{
|
|
12
|
+
* street, number, unit, postcode, locality }`), produced by `scripts/ingest-overture-addresses.ts
|
|
13
|
+
* --corpus-jsonl` (which does the DuckDB / S3 heavy lifting and flattens `address_levels` → the
|
|
14
|
+
* municipality locality). The split keeps `@mailwoman/corpus` — a RUNTIME dep of the `mailwoman`
|
|
15
|
+
* CLI — free of the heavy native `@duckdb/node-api`; the adapter just streams JSONL line-by-line,
|
|
16
|
+
* exactly like `openaddresses`.
|
|
17
|
+
*
|
|
18
|
+
* The `street` surface carries the locale's street keyword verbatim (`"CALLE JULAN"`, `"VIA
|
|
19
|
+
* ROMA"`). We map it to `street` whole and let the downstream affix-relabel split `street_prefix`
|
|
20
|
+
* — the same path every other source rides. This shard exists because the model was
|
|
21
|
+
* en-us/fr-trained and never saw non-en/fr street formats (the 2026-06-19 EU parse-blocker
|
|
22
|
+
* measured loc-correct ES 21% / IT 59% / NL 64% vs FR/US ~98%).
|
|
23
|
+
*
|
|
24
|
+
* `--country` is REQUIRED (the JSONL is per-country and the rows omit a country field), matching
|
|
25
|
+
* `openaddresses`. License is Overture's CDLA-Permissive-2.0 (attribution; not share-alike).
|
|
26
|
+
*
|
|
27
|
+
* | Field | ComponentTag | | --------- | ---------------------------------------------- | |
|
|
28
|
+
* `street` | `street` (keyword incl.; affix-relabel splits prefix) | | `number` | `house_number`
|
|
29
|
+
* (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
|
|
30
|
+
* `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import { TextSpliterator } from "spliterator"
|
|
34
|
+
|
|
35
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
36
|
+
import { formatAddress, reconcileComponents } from "../../format.ts"
|
|
37
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
38
|
+
|
|
39
|
+
export const OVERTURE_ADAPTER_ID = "overture"
|
|
40
|
+
export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
|
|
41
|
+
|
|
42
|
+
/** The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`. */
|
|
43
|
+
interface OvertureCorpusRow {
|
|
44
|
+
street?: string
|
|
45
|
+
number?: string
|
|
46
|
+
unit?: string
|
|
47
|
+
postcode?: string
|
|
48
|
+
locality?: string
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function parseLine(line: string): OvertureCorpusRow | null {
|
|
52
|
+
const t = line.trim()
|
|
53
|
+
|
|
54
|
+
if (!t || t.startsWith("#")) return null
|
|
55
|
+
|
|
56
|
+
try {
|
|
57
|
+
const o = JSON.parse(t)
|
|
58
|
+
|
|
59
|
+
return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
|
|
60
|
+
} catch {
|
|
61
|
+
return null
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export function createOvertureAdapter(): CorpusAdapter {
|
|
66
|
+
return {
|
|
67
|
+
id: OVERTURE_ADAPTER_ID,
|
|
68
|
+
defaultLicense: OVERTURE_DEFAULT_LICENSE,
|
|
69
|
+
description: "Overture Maps Addresses (global): per-country JSONL of street/number/postcode/locality.",
|
|
70
|
+
|
|
71
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
72
|
+
if (!opts.country) {
|
|
73
|
+
throw new Error(
|
|
74
|
+
"overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
|
|
75
|
+
)
|
|
76
|
+
}
|
|
77
|
+
const country = opts.country
|
|
78
|
+
|
|
79
|
+
// TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
|
|
80
|
+
// lines by returning null); the path string lets the lib own + dispose the file handle,
|
|
81
|
+
// including on an early `break`.
|
|
82
|
+
const lines = TextSpliterator.fromAsync(opts.inputPath)
|
|
83
|
+
|
|
84
|
+
let emitted = 0
|
|
85
|
+
|
|
86
|
+
for await (const line of lines) {
|
|
87
|
+
if (opts.signal?.aborted) break
|
|
88
|
+
|
|
89
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
90
|
+
|
|
91
|
+
const r = parseLine(line)
|
|
92
|
+
|
|
93
|
+
if (!r) continue
|
|
94
|
+
|
|
95
|
+
const street = r.street?.trim() ?? ""
|
|
96
|
+
const number = r.number?.trim() ?? ""
|
|
97
|
+
const unit = r.unit?.trim() ?? ""
|
|
98
|
+
const postcode = r.postcode?.trim() ?? ""
|
|
99
|
+
const locality = r.locality?.trim() ?? ""
|
|
100
|
+
|
|
101
|
+
// Only useful with a street + (postcode OR locality); point-only rows quarantine anyway.
|
|
102
|
+
if (!street) continue
|
|
103
|
+
|
|
104
|
+
if (!postcode && !locality) continue
|
|
105
|
+
|
|
106
|
+
const components: CanonicalRow["components"] = {}
|
|
107
|
+
|
|
108
|
+
// Overture "S-N" / "S/N" = sin número; only keep a real numeric house number.
|
|
109
|
+
if (/^\d/.test(number)) {
|
|
110
|
+
components.house_number = number
|
|
111
|
+
}
|
|
112
|
+
components.street = street
|
|
113
|
+
|
|
114
|
+
if (unit) {
|
|
115
|
+
components.unit = unit
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
if (postcode) {
|
|
119
|
+
components.postcode = postcode
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
if (locality) {
|
|
123
|
+
components.locality = locality
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const raw = formatAddress(components, country, { separator: ", " })
|
|
127
|
+
|
|
128
|
+
if (!raw) continue
|
|
129
|
+
|
|
130
|
+
const aligned = reconcileComponents(components, raw)
|
|
131
|
+
|
|
132
|
+
if (Object.keys(aligned).length === 0) continue
|
|
133
|
+
|
|
134
|
+
yield {
|
|
135
|
+
raw,
|
|
136
|
+
components: aligned,
|
|
137
|
+
country,
|
|
138
|
+
source: OVERTURE_ADAPTER_ID,
|
|
139
|
+
source_id: stableSourceID(OVERTURE_ADAPTER_ID, aligned),
|
|
140
|
+
corpus_version: "",
|
|
141
|
+
license: OVERTURE_DEFAULT_LICENSE,
|
|
142
|
+
}
|
|
143
|
+
emitted++
|
|
144
|
+
}
|
|
145
|
+
},
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export const overtureAdapter = createOvertureAdapter()
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `state-hi-schools`: Hawaii DOE public + charter schools CSV consumer.
|
|
7
|
+
*
|
|
8
|
+
* The Hawaii State Department of Education publishes a directory of all HIDOE schools and public
|
|
9
|
+
* charter schools (PCS) as an XLSX workbook (`SchoolList.xlsx`) with two sheets: `HIDOE` (~258
|
|
10
|
+
* rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row carries a school name,
|
|
11
|
+
* single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
|
|
12
|
+
* (complex, complex_area, district, island, charter).
|
|
13
|
+
*
|
|
14
|
+
* The adapter consumes a flat CSV the operator pre-builds via `fetch-state-hi-schools.ts`, which
|
|
15
|
+
* concatenates both sheets under one shared header. Column names match the workbook header
|
|
16
|
+
* verbatim (lower-snake-case: `code`, `name`, `address`, `city`, `zip`, ...).
|
|
17
|
+
*
|
|
18
|
+
* Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
|
|
19
|
+
* Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
|
|
20
|
+
* covers this via its optional `(?:-\d+)?` group.
|
|
21
|
+
*
|
|
22
|
+
* The `island` and `district` columns are HIDOE administrative labels (Honolulu, Central, Leeward,
|
|
23
|
+
* Windward, Hilo, Hawaii, Maui, Kauai) — they are NOT US counties and intentionally are not
|
|
24
|
+
* surfaced as `subregion`.
|
|
25
|
+
*
|
|
26
|
+
* Output: one row per school with `venue` (school name), `(house_number?, street, locality,
|
|
27
|
+
* region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
|
|
28
|
+
*
|
|
29
|
+
* License: stamped `"Public Domain"` per Hawaii state government open-data terms.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
import { createReadStream } from "node:fs"
|
|
33
|
+
|
|
34
|
+
import { parse as csvParse } from "csv-parse"
|
|
35
|
+
|
|
36
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
37
|
+
import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
38
|
+
import { reconcileComponents } from "../../format.ts"
|
|
39
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
40
|
+
|
|
41
|
+
export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
|
|
42
|
+
export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
|
|
43
|
+
|
|
44
|
+
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
45
|
+
const HI_STATE_ABBR = "HI"
|
|
46
|
+
|
|
47
|
+
interface HiSchoolRow {
|
|
48
|
+
code: string
|
|
49
|
+
name: string
|
|
50
|
+
address: string
|
|
51
|
+
city: string
|
|
52
|
+
zip: string
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function splitAddress(address: string): { house_number?: string; street: string } | null {
|
|
56
|
+
const trimmed = address.trim()
|
|
57
|
+
|
|
58
|
+
if (!trimmed) return null
|
|
59
|
+
const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
|
|
60
|
+
|
|
61
|
+
if (m) return { house_number: m[1], street: m[2]!.trim() }
|
|
62
|
+
|
|
63
|
+
return { street: trimmed }
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function normalizeZip(raw: string): string {
|
|
67
|
+
const trimmed = raw.trim()
|
|
68
|
+
|
|
69
|
+
if (!trimmed) return ""
|
|
70
|
+
|
|
71
|
+
// XLSX → CSV conversion may emit numeric ZIPs without leading zeros. HI ZIPs all begin
|
|
72
|
+
// with 96, so a 4-digit value indicates a leading-zero stripped during numeric coercion
|
|
73
|
+
// (defensive — has not been observed in the published file as of 2026-05).
|
|
74
|
+
if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
|
|
75
|
+
|
|
76
|
+
return trimmed
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
export function createStateHiSchoolsAdapter(): CorpusAdapter {
|
|
80
|
+
return {
|
|
81
|
+
id: STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
82
|
+
defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
83
|
+
description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
|
|
84
|
+
|
|
85
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
86
|
+
if (opts.country && opts.country !== "US") {
|
|
87
|
+
throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
91
|
+
const parser = stream.pipe(
|
|
92
|
+
csvParse({
|
|
93
|
+
columns: true,
|
|
94
|
+
skip_empty_lines: true,
|
|
95
|
+
relax_quotes: true,
|
|
96
|
+
relax_column_count: true,
|
|
97
|
+
})
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
const state = lookupStateAbbreviation(HI_STATE_ABBR)
|
|
101
|
+
|
|
102
|
+
if (!state) {
|
|
103
|
+
throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
let emitted = 0
|
|
107
|
+
|
|
108
|
+
try {
|
|
109
|
+
for await (const record of parser as AsyncIterable<HiSchoolRow>) {
|
|
110
|
+
if (opts.signal?.aborted) break
|
|
111
|
+
|
|
112
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
113
|
+
|
|
114
|
+
const name = (record.name ?? "").trim()
|
|
115
|
+
const address = (record.address ?? "").trim()
|
|
116
|
+
const city = (record.city ?? "").trim()
|
|
117
|
+
const zip = normalizeZip(record.zip ?? "")
|
|
118
|
+
|
|
119
|
+
if (!name || !address || !city || !zip) continue
|
|
120
|
+
|
|
121
|
+
const split = splitAddress(address)
|
|
122
|
+
|
|
123
|
+
if (!split) continue
|
|
124
|
+
|
|
125
|
+
const components: CanonicalRow["components"] = {
|
|
126
|
+
venue: name,
|
|
127
|
+
...(split.house_number ? { house_number: split.house_number } : {}),
|
|
128
|
+
street: split.street,
|
|
129
|
+
locality: city,
|
|
130
|
+
region: state.abbreviation,
|
|
131
|
+
postcode: zip,
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
135
|
+
const raw = [
|
|
136
|
+
name,
|
|
137
|
+
streetPart,
|
|
138
|
+
[city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
|
|
139
|
+
]
|
|
140
|
+
.filter(Boolean)
|
|
141
|
+
.join(", ")
|
|
142
|
+
|
|
143
|
+
const aligned = reconcileComponents(components, raw)
|
|
144
|
+
|
|
145
|
+
if (Object.keys(aligned).length <= 2) continue
|
|
146
|
+
|
|
147
|
+
const code = (record.code ?? "").toString().trim()
|
|
148
|
+
const sourceID = code
|
|
149
|
+
? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
|
|
150
|
+
: stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
|
|
151
|
+
|
|
152
|
+
yield {
|
|
153
|
+
raw,
|
|
154
|
+
components: aligned,
|
|
155
|
+
country: "US",
|
|
156
|
+
locale: "en-US",
|
|
157
|
+
source: STATE_HI_SCHOOLS_ADAPTER_ID,
|
|
158
|
+
source_id: sourceID,
|
|
159
|
+
corpus_version: "",
|
|
160
|
+
license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
|
|
161
|
+
}
|
|
162
|
+
emitted++
|
|
163
|
+
}
|
|
164
|
+
} finally {
|
|
165
|
+
stream.destroy()
|
|
166
|
+
}
|
|
167
|
+
},
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()
|