@mailwoman/corpus 7.2.0 → 7.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/package.json +59 -27
  2. package/src/adapter.ts +153 -0
  3. package/src/adapters/ban/adapter.ts +189 -0
  4. package/src/adapters/ban/street-decompose.ts +98 -0
  5. package/src/adapters/fcc-bdc/adapter.ts +188 -0
  6. package/src/adapters/geonames/adapter.ts +185 -0
  7. package/src/adapters/geonames-postal/adapter.ts +111 -0
  8. package/src/adapters/gnaf/adapter.ts +139 -0
  9. package/src/adapters/gnaf/assemble.ts +200 -0
  10. package/src/adapters/index.ts +135 -0
  11. package/src/adapters/openaddresses/adapter.ts +237 -0
  12. package/src/adapters/overture/adapter.ts +149 -0
  13. package/src/adapters/state-hi-schools/adapter.ts +171 -0
  14. package/src/adapters/state-ia-contractors/adapter.ts +148 -0
  15. package/src/adapters/state-ny-notaries/adapter.ts +169 -0
  16. package/src/adapters/state-tx-notaries/adapter.ts +160 -0
  17. package/src/adapters/synth-po-box/adapter.ts +197 -0
  18. package/src/adapters/tiger/adapter.ts +232 -0
  19. package/src/adapters/tiger/street-decompose.ts +120 -0
  20. package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
  21. package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
  22. package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
  23. package/src/adapters/usgov-nad/adapter.ts +338 -0
  24. package/src/adapters/usgov-nppes/adapter.ts +169 -0
  25. package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
  26. package/src/adapters/wof-admin-jp/adapter.ts +176 -0
  27. package/src/adapters/wof-admin-json/adapter.ts +280 -0
  28. package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
  29. package/src/align.ts +317 -0
  30. package/src/build.ts +348 -0
  31. package/src/codex/us-fips-state.ts +109 -0
  32. package/src/format.ts +14 -0
  33. package/src/golden.ts +147 -0
  34. package/src/index.ts +28 -0
  35. package/src/license.ts +48 -0
  36. package/src/parquet-wrapper/index.ts +9 -0
  37. package/src/parquet-wrapper/reader.ts +67 -0
  38. package/src/parquet-wrapper/schema.ts +80 -0
  39. package/src/parquet-wrapper/writer.ts +102 -0
  40. package/src/parquet.ts +360 -0
  41. package/src/runner.ts +283 -0
  42. package/src/shard-recipes/anchor-absorption.ts +86 -0
  43. package/src/shard-recipes/boundary-stress.ts +87 -0
  44. package/src/shard-recipes/country-balanced.ts +471 -0
  45. package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
  46. package/src/shard-recipes/fr-admin-split.ts +212 -0
  47. package/src/shard-recipes/fr-bare-street.ts +85 -0
  48. package/src/shard-recipes/fr-fragment.ts +304 -0
  49. package/src/shard-recipes/fr-order.ts +283 -0
  50. package/src/shard-recipes/german.ts +206 -0
  51. package/src/shard-recipes/house-venue.ts +78 -0
  52. package/src/shard-recipes/index.ts +76 -0
  53. package/src/shard-recipes/intersection.ts +617 -0
  54. package/src/shard-recipes/locale.ts +415 -0
  55. package/src/shard-recipes/nl-postcode.ts +105 -0
  56. package/src/shard-recipes/no-fragment.ts +206 -0
  57. package/src/shard-recipes/no-street-led.ts +130 -0
  58. package/src/shard-recipes/no-street.ts +78 -0
  59. package/src/shard-recipes/po-box-cedex.ts +853 -0
  60. package/src/shard-recipes/po-box.ts +111 -0
  61. package/src/shard-recipes/scaffold.ts +187 -0
  62. package/src/shard-recipes/si-bare-village.ts +88 -0
  63. package/src/shard-recipes/street-affix.ts +549 -0
  64. package/src/shard-recipes/street-bare.ts +90 -0
  65. package/src/shard-recipes/street.ts +74 -0
  66. package/src/shard-recipes/unit.ts +315 -0
  67. package/src/split.ts +251 -0
  68. package/src/synthesize-anchor-absorption.ts +263 -0
  69. package/src/synthesize-boundary-stress.ts +504 -0
  70. package/src/synthesize-german.ts +227 -0
  71. package/src/synthesize-house-venue.ts +192 -0
  72. package/src/synthesize-intersection.ts +205 -0
  73. package/src/synthesize-no-street.ts +341 -0
  74. package/src/synthesize-po-box.ts +307 -0
  75. package/src/synthesize-street.ts +264 -0
  76. package/src/synthesize.ts +872 -0
  77. package/src/tokenize.ts +68 -0
  78. package/src/tools/align-shard.ts +67 -0
  79. package/src/tools/audit.ts +352 -0
  80. package/src/tools/corpus-stats.ts +183 -0
  81. package/src/tools/fetch/ban.ts +288 -0
  82. package/src/tools/fetch/download.ts +119 -0
  83. package/src/tools/fetch/hrsa.ts +65 -0
  84. package/src/tools/fetch/imls-pls.ts +162 -0
  85. package/src/tools/fetch/index.ts +121 -0
  86. package/src/tools/fetch/nad.ts +304 -0
  87. package/src/tools/fetch/nppes.ts +177 -0
  88. package/src/tools/fetch/openaddresses.ts +380 -0
  89. package/src/tools/fetch/state-hi-schools.ts +217 -0
  90. package/src/tools/fetch/state-sources.ts +142 -0
  91. package/src/tools/fetch/tiger-full.ts +318 -0
  92. package/src/tools/golden-expand.ts +524 -0
  93. package/src/tools/golden-promote.ts +304 -0
  94. package/src/tools/index.ts +24 -0
  95. package/src/tools/ingest-csv.ts +420 -0
  96. package/src/tools/jsonl-to-parquet.ts +200 -0
  97. package/src/tools/lint-shard-vocab.ts +357 -0
  98. package/src/tools/lint-shard.ts +502 -0
  99. package/src/tools/overlay-manifest.ts +141 -0
  100. package/src/tools/shard-kryptonite.ts +131 -0
  101. package/src/tools/shard-translit.ts +244 -0
  102. package/src/types.ts +213 -0
  103. package/src/wof-json.ts +236 -0
@@ -0,0 +1,200 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Assemble a sampled, component-labeled Australian address set from G-NAF (the Geocoded National
7
+ * Address File — Geoscape Australia, Open G-NAF licence; any derived artifact must attribute
8
+ * "Geoscape Australia"). G-NAF is a relational PSV distribution (~16.9M addresses);
9
+ * reconstructing a street address joins three tables — ADDRESS_DETAIL (number, postcode, the
10
+ * PIDs) → STREET_LOCALITY (street name + type) → LOCALITY (suburb). State is the per-file prefix
11
+ * (ACT/NSW/…).
12
+ *
13
+ * Streaming + in-memory join via the house {@link PSVSpliterator} (pipe-separated; `mode: "object"`
14
+ * keys each row by its header) — NOT raw `read_csv` SQL, which a flat-file join doesn't need and
15
+ * which the #183–190 cleanup retired. The two lookup tables (STREET_LOCALITY ~765k rows, LOCALITY
16
+ * ~16k) fit as Maps; ADDRESS_DETAIL is streamed once and reservoir-sampled, so memory stays
17
+ * bounded (the OOM lesson from the Overture ingest).
18
+ *
19
+ * No coordinates: the output feeds the PARSER ({@link ../gnaf/adapter}, #208) — teaching the model
20
+ * AU's postcode-first / house-number-last word order, the gap `scripts/eval/au-order-probe.ts`
21
+ * pinned (65%→87% if the parse were order-robust). The parser needs the address string +
22
+ * component labels, not lat/lon.
23
+ *
24
+ * Output: component tuples as JSONL, consumed by the `gnaf` corpus adapter (which renders them in
25
+ * multiple orders + the corpus aligner BIO-labels them). An optional held-out eval set is
26
+ * excluded by (street, locality, postcode) so the training shard never overlaps the benchmark.
27
+ */
28
+
29
+ import { createWriteStream } from "node:fs"
30
+ import { readdir, readFile } from "node:fs/promises"
31
+ import { join } from "node:path"
32
+
33
+ import { PSVSpliterator } from "spliterator"
34
+
35
+ export interface GNAFAssembleOptions {
36
+ /** G-NAF `Standard` directory (holds the per-state `*_psv.psv` tables). */
37
+ standardDir: string
38
+ /** Target sample size (uniform reservoir → population-proportional across states). */
39
+ sampleSize: number
40
+ /** Output JSONL path. */
41
+ out: string
42
+ /**
43
+ * Optional held-out eval JSONL (rows with a `components` field) — its (street,locality,postcode) are excluded.
44
+ */
45
+ holdoutPath?: string
46
+ /** Progress sink (the CLI passes a setter). */
47
+ onProgress?: (message: string) => void
48
+ }
49
+
50
+ export interface GNAFAssembleResult {
51
+ written: number
52
+ seen: number
53
+ heldOut: number
54
+ byState: Record<string, number>
55
+ }
56
+
57
+ /** UPPERCASE → Title Case, preserving intra-word apostrophes/hyphens (O'Brien, Coff's Harbour). */
58
+ export function titlecase(s: string): string {
59
+ return s
60
+ .toLowerCase()
61
+ .replace(/(^|[\s'\-/])([a-z])/g, (_m, p: string, c: string) => p + c.toUpperCase())
62
+ .trim()
63
+ }
64
+
65
+ /** Holdout/dedup key: a street within a locality+postcode (house-number-agnostic — conservative). */
66
+ export function gnafHoldoutKey(street: string, locality: string, postcode: string): string {
67
+ return `${street}|${locality}|${postcode}`.toLowerCase()
68
+ }
69
+
70
+ type Row = Record<string, string | number | undefined>
71
+ async function* psvObjects(path: string): AsyncIterable<Row> {
72
+ yield* PSVSpliterator.fromAsync(path, { mode: "object", header: true }) as AsyncIterable<Row>
73
+ }
74
+
75
+ /** Load a small lookup table fully into a Map keyed by `keyCol`. */
76
+ async function loadMap<V>(paths: string[], keyCol: string, pick: (r: Row) => V): Promise<Map<string, V>> {
77
+ const m = new Map<string, V>()
78
+
79
+ for (const p of paths) {
80
+ for await (const r of psvObjects(p)) {
81
+ const k = r[keyCol]
82
+
83
+ if (k != null && k !== "") {
84
+ m.set(String(k), pick(r))
85
+ }
86
+ }
87
+ }
88
+
89
+ return m
90
+ }
91
+
92
+ /** Build the held-out key set from an eval JSONL whose rows carry a `components` object. */
93
+ async function loadHoldout(path: string): Promise<Set<string>> {
94
+ const keys = new Set<string>()
95
+ const text = await readFile(path, "utf8")
96
+
97
+ for (const line of text.split("\n")) {
98
+ if (!line.trim()) continue
99
+
100
+ try {
101
+ const c = (JSON.parse(line) as { components?: Record<string, string> }).components
102
+
103
+ if (c?.street && c?.locality && c?.postcode) {
104
+ keys.add(gnafHoldoutKey(c.street, c.locality, c.postcode))
105
+ }
106
+ } catch {
107
+ /* skip malformed */
108
+ }
109
+ }
110
+
111
+ return keys
112
+ }
113
+
114
+ export async function assembleGNAF(opts: GNAFAssembleOptions): Promise<GNAFAssembleResult> {
115
+ const progress = opts.onProgress ?? (() => {})
116
+ const files = await readdir(opts.standardDir)
117
+ const pick = (re: RegExp, exclude?: RegExp) =>
118
+ files.filter((f) => re.test(f) && !(exclude && exclude.test(f))).map((f) => join(opts.standardDir, f))
119
+
120
+ // `*_LOCALITY_psv.psv` also globs `*_STREET_LOCALITY_psv.psv` — exclude the latter explicitly.
121
+ const streetPaths = pick(/_STREET_LOCALITY_psv\.psv$/)
122
+ const localityPaths = pick(/_LOCALITY_psv\.psv$/, /_STREET_LOCALITY_psv\.psv$/)
123
+ const addressPaths = pick(/_ADDRESS_DETAIL_psv\.psv$/)
124
+
125
+ const holdout = opts.holdoutPath ? await loadHoldout(opts.holdoutPath) : new Set<string>()
126
+
127
+ if (opts.holdoutPath) {
128
+ progress(`held-out eval keys: ${holdout.size}`)
129
+ }
130
+
131
+ progress(`loading STREET_LOCALITY (${streetPaths.length} files) + LOCALITY (${localityPaths.length})…`)
132
+ const streetMap = await loadMap(streetPaths, "STREET_LOCALITY_PID", (r) => ({
133
+ name: String(r.STREET_NAME ?? ""),
134
+ type: String(r.STREET_TYPE_CODE ?? ""),
135
+ suffix: String(r.STREET_SUFFIX_CODE ?? ""),
136
+ }))
137
+ const localityMap = await loadMap(localityPaths, "LOCALITY_PID", (r) => String(r.LOCALITY_NAME ?? ""))
138
+ progress(`streets=${streetMap.size.toLocaleString()} localities=${localityMap.size.toLocaleString()}`)
139
+
140
+ const reservoir: Array<{ house_number: string; street: string; locality: string; region: string; postcode: string }> =
141
+ []
142
+ let seen = 0
143
+ let heldOut = 0
144
+
145
+ for (const p of addressPaths) {
146
+ const state = (p.match(/\/([A-Z]+)_ADDRESS_DETAIL/) ?? [])[1] ?? ""
147
+
148
+ for await (const r of psvObjects(p)) {
149
+ const numberFirst = String(r.NUMBER_FIRST ?? "")
150
+
151
+ if (!numberFirst || r.DATE_RETIRED || !r.POSTCODE) continue
152
+ const st = streetMap.get(String(r.STREET_LOCALITY_PID ?? ""))
153
+ const suburbRaw = localityMap.get(String(r.LOCALITY_PID ?? ""))
154
+
155
+ if (!st?.name || !suburbRaw) continue
156
+ const street = `${titlecase(st.name)} ${titlecase(st.type)}${st.suffix ? " " + titlecase(st.suffix) : ""}`.trim()
157
+ const locality = titlecase(suburbRaw)
158
+ const postcode = String(r.POSTCODE)
159
+
160
+ if (holdout.has(gnafHoldoutKey(street, locality, postcode))) {
161
+ heldOut++
162
+ continue
163
+ }
164
+ let house = numberFirst + (r.NUMBER_FIRST_SUFFIX ? String(r.NUMBER_FIRST_SUFFIX) : "")
165
+
166
+ if (r.NUMBER_LAST) {
167
+ house = `${house}-${String(r.NUMBER_LAST)}`
168
+ }
169
+
170
+ if (r.FLAT_NUMBER) {
171
+ house = `${String(r.FLAT_NUMBER)}/${house}`
172
+ }
173
+ const tuple = { house_number: house, street, locality, region: state, postcode }
174
+ seen++
175
+
176
+ if (reservoir.length < opts.sampleSize) {
177
+ reservoir.push(tuple)
178
+ } else {
179
+ const j = Math.floor(Math.random() * seen)
180
+
181
+ if (j < opts.sampleSize) {
182
+ reservoir[j] = tuple
183
+ }
184
+ }
185
+ }
186
+ progress(`${state}: ${seen.toLocaleString()} valid joinable seen`)
187
+ }
188
+
189
+ const out = createWriteStream(opts.out)
190
+ const byState: Record<string, number> = {}
191
+
192
+ for (const t of reservoir) {
193
+ out.write(JSON.stringify(t) + "\n")
194
+ byState[t.region] = (byState[t.region] ?? 0) + 1
195
+ }
196
+ await new Promise<void>((res) => out.end(res))
197
+ progress(`wrote ${reservoir.length.toLocaleString()} tuples → ${opts.out}`)
198
+
199
+ return { written: reservoir.length, seen, heldOut, byState }
200
+ }
@@ -0,0 +1,135 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * Adapter registry bootstrap.
7
+ *
8
+ * Importing this module registers every built-in adapter with `defaultAdapterRegistry`. The CLI
9
+ * (`commands/corpus/list.tsx`, `commands/corpus/run.tsx`) imports it once at startup.
10
+ *
11
+ * Adapters under construction live in their own subdirectories (`./wof-admin-json/`, `./ban/`, ...)
12
+ * and are added to the `BUILTIN_ADAPTERS` list here as they come online. Tests that need a
13
+ * pristine registry should construct their own `InMemoryAdapterRegistry` instead of mutating the
14
+ * default.
15
+ *
16
+ * The WOF adapters export their canonical ids — `wof-admin` and `wof-postalcode` — so existing
17
+ * `mailwoman corpus build` callsites do not need to change despite the Phase 1.5.1 SQLite →
18
+ * JSON-bundle pivot (`./wof-admin-json/` and `./wof-postalcode-json/` directories hold the
19
+ * implementations; the registered ids are unchanged).
20
+ */
21
+
22
+ import { defaultAdapterRegistry } from "../adapter.ts"
23
+ import type { CorpusAdapter } from "../types.ts"
24
+ import { banAdapter } from "./ban/adapter.ts"
25
+ import { fccBdcAdapter } from "./fcc-bdc/adapter.ts"
26
+ import { geonamesPostalAdapter } from "./geonames-postal/adapter.ts"
27
+ import { geonamesAdapter } from "./geonames/adapter.ts"
28
+ import { gnafAdapter } from "./gnaf/adapter.ts"
29
+ import { openaddressesAdapter } from "./openaddresses/adapter.ts"
30
+ import { overtureAdapter } from "./overture/adapter.ts"
31
+ import { stateHiSchoolsAdapter } from "./state-hi-schools/adapter.ts"
32
+ import { stateIaContractorsAdapter } from "./state-ia-contractors/adapter.ts"
33
+ import { stateNyNotariesAdapter } from "./state-ny-notaries/adapter.ts"
34
+ import { stateTxNotariesAdapter } from "./state-tx-notaries/adapter.ts"
35
+ import { tigerAdapter } from "./tiger/adapter.ts"
36
+ import { usgovHrsaFqhcAdapter } from "./usgov-hrsa-fqhc/adapter.ts"
37
+ import { usgovImlsPlsAdapter } from "./usgov-imls-pls/adapter.ts"
38
+ import { usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
39
+ import { usgovNADAdapter } from "./usgov-nad/adapter.ts"
40
+ import { usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
41
+ import { wofAdminAdapter } from "./wof-admin-json/adapter.ts"
42
+ import { wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
43
+
44
+ /**
45
+ * Built-in adapters. Order is significant: `corpus build` iterates this list to drive every adapter in turn.
46
+ * Coarse-first (admin → postcode), then street-level (BAN FR, TIGER US, OpenAddresses global), then adversarial-source
47
+ * (FCC BDC US, HRSA FQHC US).
48
+ *
49
+ * The `usgov-samhsa-treatment-locator` adapter is intentionally absent from this list — the SAMHSA Open Data Foundry
50
+ * bulk CSV the adapter was written against is no longer publicly distributed (see issue #33, 2026-05-17 investigation).
51
+ * The factory + named export remain available so the adapter can be hand-registered if an operator obtains a compatible
52
+ * CSV (FOIA, partner channel, upstream restoration). Re-add it here once a stable public source returns.
53
+ */
54
+ export const BUILTIN_ADAPTERS: readonly CorpusAdapter[] = [
55
+ wofAdminAdapter,
56
+ wofPostalcodeAdapter,
57
+ geonamesAdapter,
58
+ geonamesPostalAdapter,
59
+ banAdapter,
60
+ tigerAdapter,
61
+ openaddressesAdapter,
62
+ overtureAdapter,
63
+ gnafAdapter,
64
+ fccBdcAdapter,
65
+ usgovHrsaFqhcAdapter,
66
+ usgovNPPESAdapter,
67
+ usgovNADAdapter,
68
+ usgovImlsPlsAdapter,
69
+ usgovIrsBmfAdapter,
70
+ stateIaContractorsAdapter,
71
+ stateTxNotariesAdapter,
72
+ stateNyNotariesAdapter,
73
+ stateHiSchoolsAdapter,
74
+ ]
75
+
76
+ for (const adapter of BUILTIN_ADAPTERS) {
77
+ if (!defaultAdapterRegistry.get(adapter.id)) {
78
+ defaultAdapterRegistry.register(adapter)
79
+ }
80
+ }
81
+
82
+ export { BAN_ADAPTER_ID, banAdapter } from "./ban/adapter.ts"
83
+ export { FCC_BDC_ADAPTER_ID, FCC_BDC_DEFAULT_LICENSE, fccBdcAdapter } from "./fcc-bdc/adapter.ts"
84
+ export {
85
+ GEONAMES_POSTAL_ADAPTER_ID,
86
+ GEONAMES_POSTAL_DEFAULT_LICENSE,
87
+ geonamesPostalAdapter,
88
+ } from "./geonames-postal/adapter.ts"
89
+ export { GEONAMES_ADAPTER_ID, GEONAMES_DEFAULT_LICENSE, geonamesAdapter } from "./geonames/adapter.ts"
90
+ export {
91
+ OPENADDRESSES_ADAPTER_ID,
92
+ OPENADDRESSES_DEFAULT_LICENSE,
93
+ openaddressesAdapter,
94
+ } from "./openaddresses/adapter.ts"
95
+ export {
96
+ STATE_HI_SCHOOLS_ADAPTER_ID,
97
+ STATE_HI_SCHOOLS_DEFAULT_LICENSE,
98
+ stateHiSchoolsAdapter,
99
+ } from "./state-hi-schools/adapter.ts"
100
+ export {
101
+ STATE_IA_CONTRACTORS_ADAPTER_ID,
102
+ STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
103
+ stateIaContractorsAdapter,
104
+ } from "./state-ia-contractors/adapter.ts"
105
+ export {
106
+ STATE_NY_NOTARIES_ADAPTER_ID,
107
+ STATE_NY_NOTARIES_DEFAULT_LICENSE,
108
+ stateNyNotariesAdapter,
109
+ } from "./state-ny-notaries/adapter.ts"
110
+ export {
111
+ STATE_TX_NOTARIES_ADAPTER_ID,
112
+ STATE_TX_NOTARIES_DEFAULT_LICENSE,
113
+ stateTxNotariesAdapter,
114
+ } from "./state-tx-notaries/adapter.ts"
115
+ export { TIGER_ADAPTER_ID, TIGER_DEFAULT_LICENSE, tigerAdapter } from "./tiger/adapter.ts"
116
+ export {
117
+ USGOV_HRSA_FQHC_ADAPTER_ID,
118
+ USGOV_HRSA_FQHC_DEFAULT_LICENSE,
119
+ usgovHrsaFqhcAdapter,
120
+ } from "./usgov-hrsa-fqhc/adapter.ts"
121
+ export {
122
+ USGOV_IMLS_PLS_ADAPTER_ID,
123
+ USGOV_IMLS_PLS_DEFAULT_LICENSE,
124
+ usgovImlsPlsAdapter,
125
+ } from "./usgov-imls-pls/adapter.ts"
126
+ export { USGOV_IRS_BMF_ADAPTER_ID, USGOV_IRS_BMF_DEFAULT_LICENSE, usgovIrsBmfAdapter } from "./usgov-irs-bmf/adapter.ts"
127
+ export { USGOV_NAD_ADAPTER_ID, USGOV_NAD_DEFAULT_LICENSE, usgovNADAdapter } from "./usgov-nad/adapter.ts"
128
+ export { USGOV_NPPES_ADAPTER_ID, USGOV_NPPES_DEFAULT_LICENSE, usgovNPPESAdapter } from "./usgov-nppes/adapter.ts"
129
+ export {
130
+ USGOV_SAMHSA_ADAPTER_ID,
131
+ USGOV_SAMHSA_DEFAULT_LICENSE,
132
+ usgovSamhsaTreatmentLocatorAdapter,
133
+ } from "./usgov-samhsa-treatment-locator/adapter.ts"
134
+ export { WOF_ADMIN_ADAPTER_ID, wofAdminAdapter } from "./wof-admin-json/adapter.ts"
135
+ export { WOF_POSTALCODE_ADAPTER_ID, wofPostalcodeAdapter } from "./wof-postalcode-json/adapter.ts"
@@ -0,0 +1,237 @@
1
+ /**
2
+ * @copyright Sister Software
3
+ * @license AGPL-3.0
4
+ * @author Teffen Ellis, et al.
5
+ *
6
+ * `openaddresses`: Line-delimited GeoJSON adapter for openaddresses.io exports.
7
+ *
8
+ * OpenAddresses publishes country-partitioned address dumps as either CSV or line-delimited GeoJSON
9
+ * (one `Feature` per line, also called ND-GeoJSON / GeoJSONL). This adapter consumes the
10
+ * line-delimited GeoJSON shape — it streams cleanly without holding the file in memory, which
11
+ * matters for the multi-gigabyte national dumps (e.g. `us-northeast.geojsonl`, ~20M rows).
12
+ *
13
+ * The collection aggregates **hundreds** of underlying sources with **per-source licenses** (city
14
+ * open-data portals, county GIS departments, state DOTs). The adapter therefore prefers the
15
+ * per-row `LICENSE` property when present and falls back to the configured `defaultLicense`. The
16
+ * propagated license travels with each `CanonicalRow` so downstream code can stratify, exclude,
17
+ * or re-attribute by license at training time.
18
+ *
19
+ * Country must be explicit (`opts.country` REQUIRED): OpenAddresses files are organized by country
20
+ * but the row-level data doesn't include a country code, so the adapter refuses to run without
21
+ * one. This matches how a `mailwoman corpus build` invocation pins each file to a country via the
22
+ * inputs JSON.
23
+ *
24
+ * Properties consumed (per the canonical OpenAddresses schema; both UPPERCASE and lowercase
25
+ * variants are accepted because legacy dumps used UPPERCASE):
26
+ *
27
+ * | Property | ComponentTag | | ------------- |
28
+ * -------------------------------------------------------------- | | `number` | `house_number` |
29
+ * | `street` | `street` | | `unit` | `unit` (if non-empty) | | `city` | `locality` | | `region` |
30
+ * `region` (state code for US, province for CA, etc.) | | `postcode` | `postcode` | | `LICENSE` |
31
+ * per-row `license` override | | `hash` / `id` | `source_id` (prefer `hash`; fall back to `id`;
32
+ * then synthesize)|
33
+ *
34
+ * `district` is intentionally NOT mapped — for US data it carries borough or county and would
35
+ * inflate alignment quarantine because postal addresses don't include it. Phase 6+ may revisit
36
+ * for non-US locales where district names DO appear on the envelope.
37
+ */
38
+
39
+ import { TextSpliterator } from "spliterator"
40
+
41
+ import { stableSourceID } from "../../adapter.ts"
42
+ import { formatAddress, reconcileComponents } from "../../format.ts"
43
+ import { SHARE_ALIKE_PATTERN } from "../../license.ts"
44
+ import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
45
+
46
+ export const OPENADDRESSES_ADAPTER_ID = "openaddresses"
47
+ export const OPENADDRESSES_DEFAULT_LICENSE = "CC-BY-4.0"
48
+
49
+ /**
50
+ * Subset of OpenAddresses Feature properties the adapter inspects. The runtime accepts UPPERCASE or lowercase keys;
51
+ * this interface documents the canonical lowercase form after normalization.
52
+ */
53
+ interface OaProperties {
54
+ hash?: string
55
+ id?: string
56
+ number?: string
57
+ street?: string
58
+ unit?: string
59
+ city?: string
60
+ district?: string
61
+ region?: string
62
+ postcode?: string
63
+ license?: string
64
+ }
65
+
66
+ /** Return a lowercase-keyed view of a Feature's properties so case variants both work. */
67
+ function normalizeProperties(raw: unknown): OaProperties {
68
+ if (!raw || typeof raw !== "object") return {}
69
+ const out: Record<string, string> = {}
70
+
71
+ for (const [k, v] of Object.entries(raw as Record<string, unknown>)) {
72
+ if (typeof v === "string") {
73
+ out[k.toLowerCase()] = v
74
+ } else if (typeof v === "number") {
75
+ out[k.toLowerCase()] = String(v)
76
+ }
77
+ }
78
+
79
+ return out as OaProperties
80
+ }
81
+
82
+ /** Parse a single ND-GeoJSON line; return null for blanks, comments, or non-Feature shapes. */
83
+ function parseFeatureLine(line: string): OaProperties | null {
84
+ const trimmed = line.trim()
85
+
86
+ if (!trimmed || trimmed.startsWith("#")) return null
87
+ let parsed: unknown
88
+
89
+ try {
90
+ parsed = JSON.parse(trimmed)
91
+ } catch {
92
+ return null
93
+ }
94
+
95
+ if (!parsed || typeof parsed !== "object") return null
96
+ const obj = parsed as { type?: string; properties?: unknown }
97
+
98
+ if (obj.type !== "Feature") return null
99
+
100
+ return normalizeProperties(obj.properties)
101
+ }
102
+
103
+ export interface OpenaddressesAdapterOptions {
104
+ /**
105
+ * Per-row license used when a Feature lacks an explicit `LICENSE` property. Defaults to `CC-BY-4.0` — the most common
106
+ * license across the OpenAddresses collection. Override per dump via the runner's adapter-options passthrough.
107
+ */
108
+ defaultLicense?: string
109
+
110
+ /**
111
+ * Per-adapter share-alike drop. Default **true** (include) as of 2026-06-19: exclusion is a deliberate BUILD-level
112
+ * act (`buildCorpus({ excludeLicenses })` / `--exclude-share-alike`), NOT a silent adapter default (#26 — "purposely
113
+ * exclude, don't opt in to include"). Set false only for an explicit adapter-scoped drop; the build-level
114
+ * `--exclude-share-alike` is the normal path.
115
+ */
116
+ allowShareAlike?: boolean
117
+ }
118
+
119
+ /**
120
+ * Build an OpenAddresses adapter. The optional `defaultLicense` lets callers stamp a non-default fallback for dumps
121
+ * known to carry a single license throughout (e.g. a PDDL-only state slice).
122
+ */
123
+ export function createOpenaddressesAdapter(opts: OpenaddressesAdapterOptions = {}): CorpusAdapter {
124
+ const defaultLicense = opts.defaultLicense ?? OPENADDRESSES_DEFAULT_LICENSE
125
+ const allowShareAlike = opts.allowShareAlike ?? true
126
+
127
+ return {
128
+ id: OPENADDRESSES_ADAPTER_ID,
129
+ defaultLicense,
130
+ description: "OpenAddresses (global): line-delimited GeoJSON dumps with per-row licenses.",
131
+
132
+ async *rows(adapterOpts: AdapterOptions): AsyncIterable<CanonicalRow> {
133
+ if (!adapterOpts.country) {
134
+ throw new Error(
135
+ "openaddresses adapter: --country is required (OpenAddresses files are country-partitioned but rows lack a country field)"
136
+ )
137
+ }
138
+ const country = adapterOpts.country
139
+
140
+ // TextSpliterator streams string lines (parseFeatureLine keeps tolerating blank/`#`/
141
+ // malformed lines by returning null); passing the path string lets the lib own + dispose
142
+ // the file handle, including on an early `break`.
143
+ const lines = TextSpliterator.fromAsync(adapterOpts.inputPath)
144
+
145
+ let emitted = 0
146
+ let shareAlikeBlocked = 0
147
+
148
+ try {
149
+ for await (const line of lines) {
150
+ if (adapterOpts.signal?.aborted) break
151
+
152
+ if (adapterOpts.limit !== undefined && emitted >= adapterOpts.limit) break
153
+
154
+ const props = parseFeatureLine(line)
155
+
156
+ if (!props) continue
157
+
158
+ const houseNumber = props.number?.trim() ?? ""
159
+ const street = props.street?.trim() ?? ""
160
+ const unit = props.unit?.trim() ?? ""
161
+ const city = props.city?.trim() ?? ""
162
+ const region = props.region?.trim() ?? ""
163
+ const postcode = props.postcode?.trim() ?? ""
164
+
165
+ // A row is only useful if it has, at minimum, a street + (postcode OR locality).
166
+ // Pure point-only rows would land in quarantine anyway.
167
+ if (!street) continue
168
+
169
+ if (!city && !postcode) continue
170
+
171
+ const license = (props.license?.trim() || defaultLicense).trim()
172
+
173
+ if (!allowShareAlike && SHARE_ALIKE_PATTERN.test(license)) {
174
+ shareAlikeBlocked++
175
+ continue
176
+ }
177
+
178
+ const components: CanonicalRow["components"] = {}
179
+
180
+ if (houseNumber) {
181
+ components.house_number = houseNumber
182
+ }
183
+
184
+ if (street) {
185
+ components.street = street
186
+ }
187
+
188
+ if (unit) {
189
+ components.unit = unit
190
+ }
191
+
192
+ if (city) {
193
+ components.locality = city
194
+ }
195
+
196
+ if (region) {
197
+ components.region = region
198
+ }
199
+
200
+ if (postcode) {
201
+ components.postcode = postcode
202
+ }
203
+
204
+ const raw = formatAddress(components, country, { separator: ", " })
205
+
206
+ if (!raw) continue
207
+
208
+ const aligned = reconcileComponents(components, raw)
209
+
210
+ if (Object.keys(aligned).length === 0) continue
211
+
212
+ const sourceIDSeed = props.hash?.trim() || props.id?.trim()
213
+ const sourceID = sourceIDSeed
214
+ ? `${OPENADDRESSES_ADAPTER_ID}-${sourceIDSeed}`
215
+ : stableSourceID(OPENADDRESSES_ADAPTER_ID, aligned)
216
+
217
+ yield {
218
+ raw,
219
+ components: aligned,
220
+ country,
221
+ source: OPENADDRESSES_ADAPTER_ID,
222
+ source_id: sourceID,
223
+ corpus_version: "",
224
+ license,
225
+ }
226
+ emitted++
227
+ }
228
+ } finally {
229
+ if (shareAlikeBlocked > 0) {
230
+ process.stderr.write(` openaddresses: ${shareAlikeBlocked} share-alike rows dropped, ${emitted} kept\n`)
231
+ }
232
+ }
233
+ },
234
+ }
235
+ }
236
+
237
+ export const openaddressesAdapter = createOpenaddressesAdapter()