@mailwoman/corpus 7.1.0 → 7.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +15 -7
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,135 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Adapter registry bootstrap.
7
+ *
8
+ * Importing this module registers every built-in adapter with `defaultAdapterRegistry`. The CLI
9
+ * (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
10
+ *
11
+ * Adapters under construction live in their own subdirectories (`./wof-admin-json/`, `./ban/`, ...)
12
+ * and are added to the `BUILTIN_ADAPTERS` list here as they come online. Tests that need a
13
+ * pristine registry should construct their own `InMemoryAdapterRegistry` instead of mutating the
14
+ * default.
15
+ *
16
+ * The WOF adapters export their canonical ids — `wof-admin` and `wof-postalcode` — so existing
17
+ * `mailwoman corpus build` callsites do not need to change despite the Phase 1.5.1 SQLite →
18
+ * JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
19
+ * implementations; the registered ids are unchanged).
20
+ */
21
+
22
+ import { defaultAdapterRegistry } from "../adapter.ts"
23
+ import type { CorpusAdapter } from "../types.ts"
24
+ import { banAdapter } from "./ban/adapter.ts"
25
+ import { fccBdcAdapter } from "./fcc-bdc/adapter.ts"
26
+ import { geonamesPostalAdapter } from "./geonames-postal/adapter.ts"
27
+ import { geonamesAdapter } from "./geonames/adapter.ts"
28
+ import { gnafAdapter } from "./gnaf/adapter.ts"
29
+ import { openaddressesAdapter } from "./openaddresses/adapter.ts"
30
+ import { overtureAdapter } from "./overture/adapter.ts"
31
+ import { stateHiSchoolsAdapter } from "./state-hi-schools/adapter.ts"
32
+ import { stateIaContractorsAdapter } from "./state-ia-contractors/adapter.ts"
33
+ import { stateNyNotariesAdapter } from "./state-ny-notaries/adapter.ts"
34
+ import { stateTxNotariesAdapter } from "./state-tx-notaries/adapter.ts"
35
+ import { tigerAdapter } from "./tiger/adapter.ts"
36
+ import { usgovHrsaFqhcAdapter } from "./usgov-hrsa-fqhc/adapter.ts"
37
+ import { usgovImlsPlsAdapter } from "./usgov-imls-pls/adapter.ts"
38
+ import { usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
39
+ import { usgovNADAdapter } from "./usgov-nad/adapter.ts"
40
+ import { usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
41
+ import { wofAdminAdapter } from "./wof-admin-json/adapter.ts"
42
+ import { wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
43
+
44
+ /**
45
+ * Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
46
+ * Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
47
+ * (FCC BDC US, HRSA FQHC US).
48
+ *
49
+ * The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list — the SAMHSA Open Data Foundry
50
+ * bulk CSV the adapter was written against is no longer publicly distributed (see issue #33, 2026-05-17 investigation).
51
+ * The factory + named export remain available so the adapter can be hand-registered if an operator obtains a compatible
52
+ * CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
53
+ */
54
+ export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
55
+ wofAdminAdapter,
56
+ wofPostalcodeAdapter,
57
+ geonamesAdapter,
58
+ geonamesPostalAdapter,
59
+ banAdapter,
60
+ tigerAdapter,
61
+ openaddressesAdapter,
62
+ overtureAdapter,
63
+ gnafAdapter,
64
+ fccBdcAdapter,
65
+ usgovHrsaFqhcAdapter,
66
+ usgovNPPESAdapter,
67
+ usgovNADAdapter,
68
+ usgovImlsPlsAdapter,
69
+ usgovIrsBmfAdapter,
70
+ stateIaContractorsAdapter,
71
+ stateTxNotariesAdapter,
72
+ stateNyNotariesAdapter,
73
+ stateHiSchoolsAdapter,
74
+ ]
75
+
76
+ for (const adapter of BUILTIN_ADAPTERS) {
77
+ if (!defaultAdapterRegistry.get(adapter.id)) {
78
+ defaultAdapterRegistry.register(adapter)
79
+ }
80
+ }
81
+
82
+ export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
83
+ export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
84
+ export {
85
+ GEONAMES_POSTAL_ADAPTER_ID,
86
+ GEONAMES_POSTAL_DEFAULT_LICENSE,
87
+ geonamesPostalAdapter,
88
+ } from "./geonames-postal/adapter.ts"
89
+ export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
90
+ export {
91
+ OPENADDRESSES_ADAPTER_ID,
92
+ OPENADDRESSES_DEFAULT_LICENSE,
93
+ openaddressesAdapter,
94
+ } from "./openaddresses/adapter.ts"
95
+ export {
96
+ STATE_HI_SCHOOLS_ADAPTER_ID,
97
+ STATE_HI_SCHOOLS_DEFAULT_LICENSE,
98
+ stateHiSchoolsAdapter,
99
+ } from "./state-hi-schools/adapter.ts"
100
+ export {
101
+ STATE_IA_CONTRACTORS_ADAPTER_ID,
102
+ STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
103
+ stateIaContractorsAdapter,
104
+ } from "./state-ia-contractors/adapter.ts"
105
+ export {
106
+ STATE_NY_NOTARIES_ADAPTER_ID,
107
+ STATE_NY_NOTARIES_DEFAULT_LICENSE,
108
+ stateNyNotariesAdapter,
109
+ } from "./state-ny-notaries/adapter.ts"
110
+ export {
111
+ STATE_TX_NOTARIES_ADAPTER_ID,
112
+ STATE_TX_NOTARIES_DEFAULT_LICENSE,
113
+ stateTxNotariesAdapter,
114
+ } from "./state-tx-notaries/adapter.ts"
115
+ export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
116
+ export {
117
+ USGOV_HRSA_FQHC_ADAPTER_ID,
118
+ USGOV_HRSA_FQHC_DEFAULT_LICENSE,
119
+ usgovHrsaFqhcAdapter,
120
+ } from "./usgov-hrsa-fqhc/adapter.ts"
121
+ export {
122
+ USGOV_IMLS_PLS_ADAPTER_ID,
123
+ USGOV_IMLS_PLS_DEFAULT_LICENSE,
124
+ usgovImlsPlsAdapter,
125
+ } from "./usgov-imls-pls/adapter.ts"
126
+ export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
127
+ export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
128
+ export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
129
+ export {
130
+ USGOV_SAMHSA_ADAPTER_ID,
131
+ USGOV_SAMHSA_DEFAULT_LICENSE,
132
+ usgovSamhsaTreatmentLocatorAdapter,
133
+ } from "./usgov-samhsa-treatment-locator/adapter.ts"
134
+ export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
135
+ export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
@@ -0,0 +1,237 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
7
+ *
8
+ * OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
9
+ * (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
10
+ * line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
11
+ * matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
12
+ *
13
+ * The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
14
+ * open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
15
+ * per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
16
+ * propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
17
+ * or re-attribute by license at training time.
18
+ *
19
+ * Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
20
+ * but the row-level data doesn't include a country code, so the adapter refuses to run without
21
+ * one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
22
+ * inputs JSON.
23
+ *
24
+ * Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
25
+ * variants are accepted because legacy dumps used UPPERCASE):
26
+ *
27
+ * | Property | ComponentTag | | ------------- |
28
+ * -------------------------------------------------------------- | | `number` | `house_number` |
29
+ * | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
30
+ * `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
31
+ * per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
32
+ * then synthesize)|
33
+ *
34
+ * `district` is intentionally NOT mapped — for US data it carries borough or county and would
35
+ * inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
36
+ * for non-US locales where district names DO appear on the envelope.
37
+ */
38
+
39
+ import { TextSpliterator } from "spliterator"
40
+
41
+ import { stableSourceID } from "../../adapter.ts"
42
+ import { formatAddress, reconcileComponents } from "../../format.ts"
43
+ import { SHARE_ALIKE_PATTERN } from "../../license.ts"
44
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
45
+
46
+ export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
47
+ export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
48
+
49
+ /**
50
+ * Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
51
+ * this interface documents the canonical lowercase form after normalization.
52
+ */
53
+ interface OaProperties {
54
+ hash?: string
55
+ id?: string
56
+ number?: string
57
+ street?: string
58
+ unit?: string
59
+ city?: string
60
+ district?: string
61
+ region?: string
62
+ postcode?: string
63
+ license?: string
64
+ }
65
+
66
+ /** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
67
+ function normalizeProperties(raw: unknown): OaProperties {
68
+ if (!raw || typeof raw !== "object") return {}
69
+ const out: Record<string, string> = {}
70
+
71
+ for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
72
+ if (typeof v === "string") {
73
+ out[k.toLowerCase()] = v
74
+ } else if (typeof v === "number") {
75
+ out[k.toLowerCase()] = String(v)
76
+ }
77
+ }
78
+
79
+ return out as OaProperties
80
+ }
81
+
82
+ /** Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes. */
83
+ function parseFeatureLine(line: string): OaProperties | null {
84
+ const trimmed = line.trim()
85
+
86
+ if (!trimmed || trimmed.startsWith("#")) return null
87
+ let parsed: unknown
88
+
89
+ try {
90
+ parsed = JSON.parse(trimmed)
91
+ } catch {
92
+ return null
93
+ }
94
+
95
+ if (!parsed || typeof parsed !== "object") return null
96
+ const obj = parsed as { type?: string; properties?: unknown }
97
+
98
+ if (obj.type !== "Feature") return null
99
+
100
+ return normalizeProperties(obj.properties)
101
+ }
102
+
103
+ export interface OpenaddressesAdapterOptions {
104
+ /**
105
+ * Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
106
+ * license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
107
+ */
108
+ defaultLicense?: string
109
+
110
+ /**
111
+ * Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
112
+ * act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
113
+ * exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
114
+ * `--exclude-share-alike` is the normal path.
115
+ */
116
+ allowShareAlike?: boolean
117
+ }
118
+
119
+ /**
120
+ * Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
121
+ * known to carry a single license throughout (e.g. a PDDL-only state slice).
122
+ */
123
+ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
124
+ const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
125
+ const allowShareAlike = opts.allowShareAlike ?? true
126
+
127
+ return {
128
+ id: OPENADDRESSES_ADAPTER_ID,
129
+ defaultLicense,
130
+ description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
131
+
132
+ async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
133
+ if (!adapterOpts.country) {
134
+ throw new Error(
135
+ "openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
136
+ )
137
+ }
138
+ const country = adapterOpts.country
139
+
140
+ // TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
141
+ // malformed lines by returning null); passing the path string lets the lib own + dispose
142
+ // the file handle, including on an early `break`.
143
+ const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
144
+
145
+ let emitted = 0
146
+ let shareAlikeBlocked = 0
147
+
148
+ try {
149
+ for await (const line of lines) {
150
+ if (adapterOpts.signal?.aborted) break
151
+
152
+ if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
153
+
154
+ const props = parseFeatureLine(line)
155
+
156
+ if (!props) continue
157
+
158
+ const houseNumber = props.number?.trim() ?? ""
159
+ const street = props.street?.trim() ?? ""
160
+ const unit = props.unit?.trim() ?? ""
161
+ const city = props.city?.trim() ?? ""
162
+ const region = props.region?.trim() ?? ""
163
+ const postcode = props.postcode?.trim() ?? ""
164
+
165
+ // A row is only useful if it has, at minimum, a street + (postcode OR locality).
166
+ // Pure point-only rows would land in quarantine anyway.
167
+ if (!street) continue
168
+
169
+ if (!city && !postcode) continue
170
+
171
+ const license = (props.license?.trim() || defaultLicense).trim()
172
+
173
+ if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
174
+ shareAlikeBlocked++
175
+ continue
176
+ }
177
+
178
+ const components: CanonicalRow["components"] = {}
179
+
180
+ if (houseNumber) {
181
+ components.house_number = houseNumber
182
+ }
183
+
184
+ if (street) {
185
+ components.street = street
186
+ }
187
+
188
+ if (unit) {
189
+ components.unit = unit
190
+ }
191
+
192
+ if (city) {
193
+ components.locality = city
194
+ }
195
+
196
+ if (region) {
197
+ components.region = region
198
+ }
199
+
200
+ if (postcode) {
201
+ components.postcode = postcode
202
+ }
203
+
204
+ const raw = formatAddress(components, country, { separator: ", " })
205
+
206
+ if (!raw) continue
207
+
208
+ const aligned = reconcileComponents(components, raw)
209
+
210
+ if (Object.keys(aligned).length === 0) continue
211
+
212
+ const sourceIDSeed = props.hash?.trim() || props.id?.trim()
213
+ const sourceID = sourceIDSeed
214
+ ? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
215
+ : stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
216
+
217
+ yield {
218
+ raw,
219
+ components: aligned,
220
+ country,
221
+ source: OPENADDRESSES_ADAPTER_ID,
222
+ source_id: sourceID,
223
+ corpus_version: "",
224
+ license,
225
+ }
226
+ emitted++
227
+ }
228
+ } finally {
229
+ if (shareAlikeBlocked > 0) {
230
+ process.stderr.write(` openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
231
+ }
232
+ }
233
+ },
234
+ }
235
+ }
236
+
237
+ export const openaddressesAdapter = createOpenaddressesAdapter()
@@ -0,0 +1,149 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `overture`: Overture Maps Addresses adapter (epic #470 — the gated corpus adapter, realized
7
+ * 2026-06-20). Overture's global Addresses theme is the single-schema, well-normalized address
8
+ * dataset that fixes OpenAddresses' per-country patchiness (OA dropped Spain; OA-DE omits the
9
+ * Bundesland) — it even re-hosts the OA Spain data the standalone OA bucket no longer serves.
10
+ *
11
+ * This adapter consumes a per-country LINE-DELIMITED JSON dump of the corpus-relevant fields (`{
12
+ * street, number, unit, postcode, locality }`), produced by `scripts/ingest-overture-addresses.ts
13
+ * --corpus-jsonl` (which does the DuckDB / S3 heavy lifting and flattens `address_levels` → the
14
+ * municipality locality). The split keeps `@mailwoman/corpus` — a RUNTIME dep of the `mailwoman`
15
+ * CLI — free of the heavy native `@duckdb/node-api`; the adapter just streams JSONL line-by-line,
16
+ * exactly like `openaddresses`.
17
+ *
18
+ * The `street` surface carries the locale's street keyword verbatim (`"CALLE JULAN"`, `"VIA
19
+ * ROMA"`). We map it to `street` whole and let the downstream affix-relabel split `street_prefix`
20
+ * — the same path every other source rides. This shard exists because the model was
21
+ * en-us/fr-trained and never saw non-en/fr street formats (the 2026-06-19 EU parse-blocker
22
+ * measured loc-correct ES 21% / IT 59% / NL 64% vs FR/US ~98%).
23
+ *
24
+ * `--country` is REQUIRED (the JSONL is per-country and the rows omit a country field), matching
25
+ * `openaddresses`. License is Overture's CDLA-Permissive-2.0 (attribution; not share-alike).
26
+ *
27
+ * | Field | ComponentTag | | --------- | ---------------------------------------------- | |
28
+ * `street` | `street` (keyword incl.; affix-relabel splits prefix) | | `number` | `house_number`
29
+ * (skipped when "S-N"/"S/N" = sin número) | | `unit` | `unit` (if non-empty) | | `postcode`|
30
+ * `postcode` | | `locality`| `locality` (Overture address_levels municipality, or postal_city) |
31
+ */
32
+
33
+ import { TextSpliterator } from "spliterator"
34
+
35
+ import { stableSourceID } from "../../adapter.ts"
36
+ import { formatAddress, reconcileComponents } from "../../format.ts"
37
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
38
+
39
+ export const OVERTURE_ADAPTER_ID = "overture"
40
+ export const OVERTURE_DEFAULT_LICENSE = "CDLA-Permissive-2.0"
41
+
42
+ /** The flattened per-row shape emitted by `ingest-overture-addresses.ts --corpus-jsonl`. */
43
+ interface OvertureCorpusRow {
44
+ street?: string
45
+ number?: string
46
+ unit?: string
47
+ postcode?: string
48
+ locality?: string
49
+ }
50
+
51
+ function parseLine(line: string): OvertureCorpusRow | null {
52
+ const t = line.trim()
53
+
54
+ if (!t || t.startsWith("#")) return null
55
+
56
+ try {
57
+ const o = JSON.parse(t)
58
+
59
+ return o && typeof o === "object" ? (o as OvertureCorpusRow) : null
60
+ } catch {
61
+ return null
62
+ }
63
+ }
64
+
65
+ export function createOvertureAdapter(): CorpusAdapter {
66
+ return {
67
+ id: OVERTURE_ADAPTER_ID,
68
+ defaultLicense: OVERTURE_DEFAULT_LICENSE,
69
+ description: "Overture Maps Addresses (global): per-country JSONL of street/number/postcode/locality.",
70
+
71
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
72
+ if (!opts.country) {
73
+ throw new Error(
74
+ "overture adapter: --country is required (the Overture JSONL is per-country and rows omit a country field)"
75
+ )
76
+ }
77
+ const country = opts.country
78
+
79
+ // TextSpliterator streams string lines (parseLine keeps tolerating blank/`#`/malformed
80
+ // lines by returning null); the path string lets the lib own + dispose the file handle,
81
+ // including on an early `break`.
82
+ const lines = TextSpliterator.fromAsync(opts.inputPath)
83
+
84
+ let emitted = 0
85
+
86
+ for await (const line of lines) {
87
+ if (opts.signal?.aborted) break
88
+
89
+ if (opts.limit !== undefined && emitted >= opts.limit) break
90
+
91
+ const r = parseLine(line)
92
+
93
+ if (!r) continue
94
+
95
+ const street = r.street?.trim() ?? ""
96
+ const number = r.number?.trim() ?? ""
97
+ const unit = r.unit?.trim() ?? ""
98
+ const postcode = r.postcode?.trim() ?? ""
99
+ const locality = r.locality?.trim() ?? ""
100
+
101
+ // Only useful with a street + (postcode OR locality); point-only rows quarantine anyway.
102
+ if (!street) continue
103
+
104
+ if (!postcode && !locality) continue
105
+
106
+ const components: CanonicalRow["components"] = {}
107
+
108
+ // Overture "S-N" / "S/N" = sin número; only keep a real numeric house number.
109
+ if (/^\d/.test(number)) {
110
+ components.house_number = number
111
+ }
112
+ components.street = street
113
+
114
+ if (unit) {
115
+ components.unit = unit
116
+ }
117
+
118
+ if (postcode) {
119
+ components.postcode = postcode
120
+ }
121
+
122
+ if (locality) {
123
+ components.locality = locality
124
+ }
125
+
126
+ const raw = formatAddress(components, country, { separator: ", " })
127
+
128
+ if (!raw) continue
129
+
130
+ const aligned = reconcileComponents(components, raw)
131
+
132
+ if (Object.keys(aligned).length === 0) continue
133
+
134
+ yield {
135
+ raw,
136
+ components: aligned,
137
+ country,
138
+ source: OVERTURE_ADAPTER_ID,
139
+ source_id: stableSourceID(OVERTURE_ADAPTER_ID, aligned),
140
+ corpus_version: "",
141
+ license: OVERTURE_DEFAULT_LICENSE,
142
+ }
143
+ emitted++
144
+ }
145
+ },
146
+ }
147
+ }
148
+
149
+ export const overtureAdapter = createOvertureAdapter()
@@ -0,0 +1,171 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `state-hi-schools`: Hawaii DOE public + charter schools CSV consumer.
7
+ *
8
+ * The Hawaii State Department of Education publishes a directory of all HIDOE schools and public
9
+ * charter schools (PCS) as an XLSX workbook (`SchoolList.xlsx`) with two sheets: `HIDOE` (~258
10
+ * rows) and `PCS` (~38 rows). Total ~296 rows statewide. Each row carries a school name,
11
+ * single-line street address, city, ZIP, a numeric `code`, and HI-specific administrative columns
12
+ * (complex, complex_area, district, island, charter).
13
+ *
14
+ * The adapter consumes a flat CSV the operator pre-builds via `fetch-state-hi-schools.ts`, which
15
+ * concatenates both sheets under one shared header. Column names match the workbook header
16
+ * verbatim (lower-snake-case: `code`, `name`, `address`, `city`, `zip`, ...).
17
+ *
18
+ * Address parsing notes: Hawaii's residential numbering is hyphenated on Oahu (`47-470 Hui Aeko
19
+ * Place`), Kauai (`2-4035 Kaumualii Hwy`), and elsewhere. The shared HOUSE_NUMBER_PREFIX regex
20
+ * covers this via its optional `(?:-\d+)?` group.
21
+ *
22
+ * The `island` and `district` columns are HIDOE administrative labels (Honolulu, Central, Leeward,
23
+ * Windward, Hilo, Hawaii, Maui, Kauai) — they are NOT US counties and intentionally are not
24
+ * surfaced as `subregion`.
25
+ *
26
+ * Output: one row per school with `venue` (school name), `(house_number?, street, locality,
27
+ * region=HI, postcode)`, and a stable `source_id` derived from the school `code`.
28
+ *
29
+ * License: stamped `"Public Domain"` per Hawaii state government open-data terms.
30
+ */
31
+
32
+ import { createReadStream } from "node:fs"
33
+
34
+ import { parse as csvParse } from "csv-parse"
35
+
36
+ import { stableSourceID } from "../../adapter.ts"
37
+ import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
38
+ import { reconcileComponents } from "../../format.ts"
39
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
40
+
41
+ export const STATE_HI_SCHOOLS_ADAPTER_ID = "state-hi-schools"
42
+ export const STATE_HI_SCHOOLS_DEFAULT_LICENSE = "Public Domain"
43
+
44
+ const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
45
+ const HI_STATE_ABBR = "HI"
46
+
47
+ interface HiSchoolRow {
48
+ code: string
49
+ name: string
50
+ address: string
51
+ city: string
52
+ zip: string
53
+ }
54
+
55
+ function splitAddress(address: string): { house_number?: string; street: string } | null {
56
+ const trimmed = address.trim()
57
+
58
+ if (!trimmed) return null
59
+ const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
60
+
61
+ if (m) return { house_number: m[1], street: m[2]!.trim() }
62
+
63
+ return { street: trimmed }
64
+ }
65
+
66
+ function normalizeZip(raw: string): string {
67
+ const trimmed = raw.trim()
68
+
69
+ if (!trimmed) return ""
70
+
71
+ // XLSX → CSV conversion may emit numeric ZIPs without leading zeros. HI ZIPs all begin
72
+ // with 96, so a 4-digit value indicates a leading-zero stripped during numeric coercion
73
+ // (defensive — has not been observed in the published file as of 2026-05).
74
+ if (/^\d{4}$/.test(trimmed)) return `0${trimmed}`
75
+
76
+ return trimmed
77
+ }
78
+
79
+ export function createStateHiSchoolsAdapter(): CorpusAdapter {
80
+ return {
81
+ id: STATE_HI_SCHOOLS_ADAPTER_ID,
82
+ defaultLicense: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
83
+ description: "Hawaii DOE School Directory — ~300 K-12 public + charter schools with venue+address (public-domain).",
84
+
85
+ async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
86
+ if (opts.country && opts.country !== "US") {
87
+ throw new Error(`state-hi-schools adapter: only US supported, got country=${opts.country}`)
88
+ }
89
+
90
+ const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
91
+ const parser = stream.pipe(
92
+ csvParse({
93
+ columns: true,
94
+ skip_empty_lines: true,
95
+ relax_quotes: true,
96
+ relax_column_count: true,
97
+ })
98
+ )
99
+
100
+ const state = lookupStateAbbreviation(HI_STATE_ABBR)
101
+
102
+ if (!state) {
103
+ throw new Error(`state-hi-schools adapter: HI not found in state codex (corpus bug)`)
104
+ }
105
+
106
+ let emitted = 0
107
+
108
+ try {
109
+ for await (const record of parser as AsyncIterable<HiSchoolRow>) {
110
+ if (opts.signal?.aborted) break
111
+
112
+ if (opts.limit !== undefined && emitted >= opts.limit) break
113
+
114
+ const name = (record.name ?? "").trim()
115
+ const address = (record.address ?? "").trim()
116
+ const city = (record.city ?? "").trim()
117
+ const zip = normalizeZip(record.zip ?? "")
118
+
119
+ if (!name || !address || !city || !zip) continue
120
+
121
+ const split = splitAddress(address)
122
+
123
+ if (!split) continue
124
+
125
+ const components: CanonicalRow["components"] = {
126
+ venue: name,
127
+ ...(split.house_number ? { house_number: split.house_number } : {}),
128
+ street: split.street,
129
+ locality: city,
130
+ region: state.abbreviation,
131
+ postcode: zip,
132
+ }
133
+
134
+ const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
135
+ const raw = [
136
+ name,
137
+ streetPart,
138
+ [city, [HI_STATE_ABBR, zip].filter(Boolean).join(" ")].filter(Boolean).join(", "),
139
+ ]
140
+ .filter(Boolean)
141
+ .join(", ")
142
+
143
+ const aligned = reconcileComponents(components, raw)
144
+
145
+ if (Object.keys(aligned).length <= 2) continue
146
+
147
+ const code = (record.code ?? "").toString().trim()
148
+ const sourceID = code
149
+ ? `${STATE_HI_SCHOOLS_ADAPTER_ID}-${code}`
150
+ : stableSourceID(STATE_HI_SCHOOLS_ADAPTER_ID, aligned)
151
+
152
+ yield {
153
+ raw,
154
+ components: aligned,
155
+ country: "US",
156
+ locale: "en-US",
157
+ source: STATE_HI_SCHOOLS_ADAPTER_ID,
158
+ source_id: sourceID,
159
+ corpus_version: "",
160
+ license: STATE_HI_SCHOOLS_DEFAULT_LICENSE,
161
+ }
162
+ emitted++
163
+ }
164
+ } finally {
165
+ stream.destroy()
166
+ }
167
+ },
168
+ }
169
+ }
170
+
171
+ export const stateHiSchoolsAdapter = createStateHiSchoolsAdapter()