@mailwoman/corpus 7.1.0 → 7.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +15 -7
- package/src/adapter.ts +153 -0
- package/src/adapters/ban/adapter.ts +189 -0
- package/src/adapters/ban/street-decompose.ts +98 -0
- package/src/adapters/fcc-bdc/adapter.ts +188 -0
- package/src/adapters/geonames/adapter.ts +185 -0
- package/src/adapters/geonames-postal/adapter.ts +111 -0
- package/src/adapters/gnaf/adapter.ts +139 -0
- package/src/adapters/gnaf/assemble.ts +200 -0
- package/src/adapters/index.ts +135 -0
- package/src/adapters/openaddresses/adapter.ts +237 -0
- package/src/adapters/overture/adapter.ts +149 -0
- package/src/adapters/state-hi-schools/adapter.ts +171 -0
- package/src/adapters/state-ia-contractors/adapter.ts +148 -0
- package/src/adapters/state-ny-notaries/adapter.ts +169 -0
- package/src/adapters/state-tx-notaries/adapter.ts +160 -0
- package/src/adapters/synth-po-box/adapter.ts +197 -0
- package/src/adapters/tiger/adapter.ts +232 -0
- package/src/adapters/tiger/street-decompose.ts +120 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +189 -0
- package/src/adapters/usgov-imls-pls/adapter.ts +152 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +155 -0
- package/src/adapters/usgov-nad/adapter.ts +338 -0
- package/src/adapters/usgov-nppes/adapter.ts +169 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +211 -0
- package/src/adapters/wof-admin-jp/adapter.ts +176 -0
- package/src/adapters/wof-admin-json/adapter.ts +280 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +209 -0
- package/src/align.ts +317 -0
- package/src/build.ts +348 -0
- package/src/codex/us-fips-state.ts +109 -0
- package/src/format.ts +14 -0
- package/src/golden.ts +147 -0
- package/src/index.ts +28 -0
- package/src/license.ts +48 -0
- package/src/parquet-wrapper/index.ts +9 -0
- package/src/parquet-wrapper/reader.ts +67 -0
- package/src/parquet-wrapper/schema.ts +80 -0
- package/src/parquet-wrapper/writer.ts +102 -0
- package/src/parquet.ts +360 -0
- package/src/runner.ts +283 -0
- package/src/shard-recipes/anchor-absorption.ts +86 -0
- package/src/shard-recipes/boundary-stress.ts +87 -0
- package/src/shard-recipes/country-balanced.ts +471 -0
- package/src/shard-recipes/cz-pcfirst-preposition.ts +83 -0
- package/src/shard-recipes/fr-admin-split.ts +212 -0
- package/src/shard-recipes/fr-bare-street.ts +85 -0
- package/src/shard-recipes/fr-fragment.ts +304 -0
- package/src/shard-recipes/fr-order.ts +283 -0
- package/src/shard-recipes/german.ts +206 -0
- package/src/shard-recipes/house-venue.ts +78 -0
- package/src/shard-recipes/index.ts +76 -0
- package/src/shard-recipes/intersection.ts +617 -0
- package/src/shard-recipes/locale.ts +415 -0
- package/src/shard-recipes/nl-postcode.ts +105 -0
- package/src/shard-recipes/no-fragment.ts +206 -0
- package/src/shard-recipes/no-street-led.ts +130 -0
- package/src/shard-recipes/no-street.ts +78 -0
- package/src/shard-recipes/po-box-cedex.ts +853 -0
- package/src/shard-recipes/po-box.ts +111 -0
- package/src/shard-recipes/scaffold.ts +187 -0
- package/src/shard-recipes/si-bare-village.ts +88 -0
- package/src/shard-recipes/street-affix.ts +549 -0
- package/src/shard-recipes/street-bare.ts +90 -0
- package/src/shard-recipes/street.ts +74 -0
- package/src/shard-recipes/unit.ts +315 -0
- package/src/split.ts +251 -0
- package/src/synthesize-anchor-absorption.ts +263 -0
- package/src/synthesize-boundary-stress.ts +504 -0
- package/src/synthesize-german.ts +227 -0
- package/src/synthesize-house-venue.ts +192 -0
- package/src/synthesize-intersection.ts +205 -0
- package/src/synthesize-no-street.ts +341 -0
- package/src/synthesize-po-box.ts +307 -0
- package/src/synthesize-street.ts +264 -0
- package/src/synthesize.ts +872 -0
- package/src/tokenize.ts +68 -0
- package/src/tools/align-shard.ts +67 -0
- package/src/tools/audit.ts +352 -0
- package/src/tools/corpus-stats.ts +183 -0
- package/src/tools/fetch/ban.ts +288 -0
- package/src/tools/fetch/download.ts +119 -0
- package/src/tools/fetch/hrsa.ts +65 -0
- package/src/tools/fetch/imls-pls.ts +162 -0
- package/src/tools/fetch/index.ts +121 -0
- package/src/tools/fetch/nad.ts +304 -0
- package/src/tools/fetch/nppes.ts +177 -0
- package/src/tools/fetch/openaddresses.ts +380 -0
- package/src/tools/fetch/state-hi-schools.ts +217 -0
- package/src/tools/fetch/state-sources.ts +142 -0
- package/src/tools/fetch/tiger-full.ts +318 -0
- package/src/tools/golden-expand.ts +524 -0
- package/src/tools/golden-promote.ts +304 -0
- package/src/tools/index.ts +24 -0
- package/src/tools/ingest-csv.ts +420 -0
- package/src/tools/jsonl-to-parquet.ts +200 -0
- package/src/tools/lint-shard-vocab.ts +357 -0
- package/src/tools/lint-shard.ts +502 -0
- package/src/tools/overlay-manifest.ts +141 -0
- package/src/tools/shard-kryptonite.ts +131 -0
- package/src/tools/shard-translit.ts +244 -0
- package/src/types.ts +213 -0
- package/src/wof-json.ts +236 -0
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `state-ia-contractors`: Iowa Active Construction Contractor Registrations CSV consumer.
|
|
7
|
+
*
|
|
8
|
+
* Iowa Workforce Development publishes a public registry of active construction contractors. Each
|
|
9
|
+
* row carries a business name, street address, city/state/zip, and contact info.
|
|
10
|
+
*
|
|
11
|
+
* The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`.
|
|
12
|
+
*
|
|
13
|
+
* Output: one row per contractor with `venue` (business name) and address quad `(house_number,
|
|
14
|
+
* street, locality, region, postcode)`.
|
|
15
|
+
*
|
|
16
|
+
* License: stamped `"Public Domain"` per Iowa state government open-data terms.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
import { createReadStream } from "node:fs"
|
|
20
|
+
|
|
21
|
+
import { parse as csvParse } from "csv-parse"
|
|
22
|
+
|
|
23
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
24
|
+
import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
25
|
+
import { reconcileComponents } from "../../format.ts"
|
|
26
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
27
|
+
|
|
28
|
+
export const STATE_IA_CONTRACTORS_ADAPTER_ID = "state-ia-contractors"
|
|
29
|
+
export const STATE_IA_CONTRACTORS_DEFAULT_LICENSE = "Public Domain"
|
|
30
|
+
|
|
31
|
+
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
32
|
+
|
|
33
|
+
interface IaContractorRow {
|
|
34
|
+
"Registration #": string
|
|
35
|
+
"Business Name": string
|
|
36
|
+
"Address 1": string
|
|
37
|
+
"Address 2": string
|
|
38
|
+
City: string
|
|
39
|
+
State: string
|
|
40
|
+
"Zip Code": string
|
|
41
|
+
"First Name": string
|
|
42
|
+
"Last Name": string
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
function splitAddress(address: string): { house_number?: string; street: string } | null {
|
|
46
|
+
const trimmed = address.trim()
|
|
47
|
+
|
|
48
|
+
if (!trimmed) return null
|
|
49
|
+
const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
|
|
50
|
+
|
|
51
|
+
if (m) return { house_number: m[1], street: m[2]!.trim() }
|
|
52
|
+
|
|
53
|
+
return { street: trimmed }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function createStateIaContractorsAdapter(): CorpusAdapter {
|
|
57
|
+
return {
|
|
58
|
+
id: STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
59
|
+
defaultLicense: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
60
|
+
description:
|
|
61
|
+
"Iowa Active Construction Contractor Registrations — business name + full street address (public-domain).",
|
|
62
|
+
|
|
63
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
64
|
+
if (opts.country && opts.country !== "US") {
|
|
65
|
+
throw new Error(`state-ia-contractors adapter: only US supported, got country=${opts.country}`)
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
69
|
+
const parser = stream.pipe(
|
|
70
|
+
csvParse({
|
|
71
|
+
columns: true,
|
|
72
|
+
skip_empty_lines: true,
|
|
73
|
+
relax_quotes: true,
|
|
74
|
+
relax_column_count: true,
|
|
75
|
+
})
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
let emitted = 0
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
for await (const record of parser as AsyncIterable<IaContractorRow>) {
|
|
82
|
+
if (opts.signal?.aborted) break
|
|
83
|
+
|
|
84
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
85
|
+
|
|
86
|
+
const businessName = (record["Business Name"] ?? "").trim()
|
|
87
|
+
const address1 = (record["Address 1"] ?? "").trim()
|
|
88
|
+
const address2 = (record["Address 2"] ?? "").trim()
|
|
89
|
+
const city = (record.City ?? "").trim()
|
|
90
|
+
const stateAbbr = (record.State ?? "").trim()
|
|
91
|
+
const zip = (record["Zip Code"] ?? "").trim()
|
|
92
|
+
|
|
93
|
+
if (!city || !zip) continue
|
|
94
|
+
|
|
95
|
+
const state = lookupStateAbbreviation(stateAbbr)
|
|
96
|
+
|
|
97
|
+
if (!state) continue
|
|
98
|
+
|
|
99
|
+
const fullAddress = [address1, address2].filter(Boolean).join(" ")
|
|
100
|
+
const split = splitAddress(fullAddress)
|
|
101
|
+
|
|
102
|
+
if (!split) continue
|
|
103
|
+
|
|
104
|
+
const venue = businessName || undefined
|
|
105
|
+
|
|
106
|
+
const components: CanonicalRow["components"] = {
|
|
107
|
+
...(venue ? { venue } : {}),
|
|
108
|
+
...(split.house_number ? { house_number: split.house_number } : {}),
|
|
109
|
+
street: split.street,
|
|
110
|
+
locality: city,
|
|
111
|
+
region: state.abbreviation,
|
|
112
|
+
postcode: zip,
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
116
|
+
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
117
|
+
.filter(Boolean)
|
|
118
|
+
.join(", ")
|
|
119
|
+
|
|
120
|
+
const aligned = reconcileComponents(components, raw)
|
|
121
|
+
|
|
122
|
+
if (Object.keys(aligned).length <= 2) continue
|
|
123
|
+
|
|
124
|
+
const regNum = (record["Registration #"] ?? "").trim()
|
|
125
|
+
const sourceID = regNum
|
|
126
|
+
? `${STATE_IA_CONTRACTORS_ADAPTER_ID}-${regNum}`
|
|
127
|
+
: stableSourceID(STATE_IA_CONTRACTORS_ADAPTER_ID, aligned)
|
|
128
|
+
|
|
129
|
+
yield {
|
|
130
|
+
raw,
|
|
131
|
+
components: aligned,
|
|
132
|
+
country: "US",
|
|
133
|
+
locale: "en-US",
|
|
134
|
+
source: STATE_IA_CONTRACTORS_ADAPTER_ID,
|
|
135
|
+
source_id: sourceID,
|
|
136
|
+
corpus_version: "",
|
|
137
|
+
license: STATE_IA_CONTRACTORS_DEFAULT_LICENSE,
|
|
138
|
+
}
|
|
139
|
+
emitted++
|
|
140
|
+
}
|
|
141
|
+
} finally {
|
|
142
|
+
stream.destroy()
|
|
143
|
+
}
|
|
144
|
+
},
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export const stateIaContractorsAdapter = createStateIaContractorsAdapter()
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `state-ny-notaries`: New York Commissioned Notaries CSV consumer.
|
|
7
|
+
*
|
|
8
|
+
* The New York Department of State publishes a registry of commissioned notaries public. Each row
|
|
9
|
+
* optionally carries a business name and business address (~1-5% fill rate).
|
|
10
|
+
*
|
|
11
|
+
* The adapter consumes the CSV the operator pre-downloads via `fetch-state-sources.ts`. Column
|
|
12
|
+
* names match the data.ny.gov export header (note: some columns have leading spaces).
|
|
13
|
+
*
|
|
14
|
+
* License: stamped `"Public Domain"` per New York state government open-data terms.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { createReadStream } from "node:fs"
|
|
18
|
+
|
|
19
|
+
import { parse as csvParse } from "csv-parse"
|
|
20
|
+
|
|
21
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
22
|
+
import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
23
|
+
import { reconcileComponents } from "../../format.ts"
|
|
24
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
25
|
+
|
|
26
|
+
export const STATE_NY_NOTARIES_ADAPTER_ID = "state-ny-notaries"
|
|
27
|
+
export const STATE_NY_NOTARIES_DEFAULT_LICENSE = "Public Domain"
|
|
28
|
+
|
|
29
|
+
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
30
|
+
|
|
31
|
+
interface NyNotaryRow {
|
|
32
|
+
"Commission Holder Name": string
|
|
33
|
+
"Commission Number (UID)": string
|
|
34
|
+
"Business Name (if available)": string
|
|
35
|
+
"Business Address 1 (if available)": string
|
|
36
|
+
"Business Address 2 (if available)": string
|
|
37
|
+
" Business City (if available)": string
|
|
38
|
+
"Business State (if available)": string
|
|
39
|
+
"Business Zip (if available)": string
|
|
40
|
+
"Commissioned County": string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function splitAddress(address: string): { house_number?: string; street: string } | null {
|
|
44
|
+
const trimmed = address.trim()
|
|
45
|
+
|
|
46
|
+
if (!trimmed) return null
|
|
47
|
+
const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
|
|
48
|
+
|
|
49
|
+
if (m) return { house_number: m[1], street: m[2]!.trim() }
|
|
50
|
+
|
|
51
|
+
return { street: trimmed }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const RAW_NY_COLUMNS = [
|
|
55
|
+
"Commission Holder Name",
|
|
56
|
+
"Commission Number (UID)",
|
|
57
|
+
"Business Name (if available)",
|
|
58
|
+
"Business Address 1 (if available)",
|
|
59
|
+
"Business Address 2 (if available)",
|
|
60
|
+
" Business City (if available)",
|
|
61
|
+
"Business State (if available)",
|
|
62
|
+
"Business Zip (if available)",
|
|
63
|
+
"Commissioned County",
|
|
64
|
+
] as const
|
|
65
|
+
|
|
66
|
+
export function createStateNyNotariesAdapter(): CorpusAdapter {
|
|
67
|
+
return {
|
|
68
|
+
id: STATE_NY_NOTARIES_ADAPTER_ID,
|
|
69
|
+
defaultLicense: STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
70
|
+
description: "New York Commissioned Notaries — name + optional business address (public-domain).",
|
|
71
|
+
|
|
72
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
73
|
+
if (opts.country && opts.country !== "US") {
|
|
74
|
+
throw new Error(`state-ny-notaries adapter: only US supported, got country=${opts.country}`)
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
78
|
+
const parser = stream.pipe(
|
|
79
|
+
csvParse({
|
|
80
|
+
columns: true,
|
|
81
|
+
skip_empty_lines: true,
|
|
82
|
+
relax_quotes: true,
|
|
83
|
+
relax_column_count: true,
|
|
84
|
+
})
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
let emitted = 0
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
for await (const rawRecord of parser as AsyncIterable<Record<string, string>>) {
|
|
91
|
+
if (opts.signal?.aborted) break
|
|
92
|
+
|
|
93
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
94
|
+
|
|
95
|
+
// NY CSV has columns with leading spaces, so we normalize by trimming keys.
|
|
96
|
+
const record: Record<string, string> = {}
|
|
97
|
+
|
|
98
|
+
for (const key of Object.keys(rawRecord)) {
|
|
99
|
+
record[key.trim()] = rawRecord[key] ?? ""
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const holderName = (record["Commission Holder Name"] ?? "").trim()
|
|
103
|
+
const businessName = (record["Business Name (if available)"] ?? "").trim()
|
|
104
|
+
const address1 = (record["Business Address 1 (if available)"] ?? "").trim()
|
|
105
|
+
const address2 = (record["Business Address 2 (if available)"] ?? "").trim()
|
|
106
|
+
const city = (record["Business City (if available)"] ?? "").trim()
|
|
107
|
+
const stateAbbr = (record["Business State (if available)"] ?? "").trim()
|
|
108
|
+
const zip = (record["Business Zip (if available)"] ?? "").trim()
|
|
109
|
+
const county = (record["Commissioned County"] ?? "").trim()
|
|
110
|
+
|
|
111
|
+
if (!city || !stateAbbr || !zip) continue
|
|
112
|
+
|
|
113
|
+
if (!address1 && !address2) continue
|
|
114
|
+
|
|
115
|
+
const state = lookupStateAbbreviation(stateAbbr)
|
|
116
|
+
|
|
117
|
+
if (!state) continue
|
|
118
|
+
|
|
119
|
+
const fullAddress = [address1, address2].filter(Boolean).join(" ")
|
|
120
|
+
const split = splitAddress(fullAddress)
|
|
121
|
+
|
|
122
|
+
if (!split) continue
|
|
123
|
+
|
|
124
|
+
const venue = businessName || holderName || undefined
|
|
125
|
+
|
|
126
|
+
const components: CanonicalRow["components"] = {
|
|
127
|
+
...(venue ? { venue } : {}),
|
|
128
|
+
...(split.house_number ? { house_number: split.house_number } : {}),
|
|
129
|
+
street: split.street,
|
|
130
|
+
locality: city,
|
|
131
|
+
region: state.abbreviation,
|
|
132
|
+
postcode: zip,
|
|
133
|
+
...(county ? { subregion: county } : {}),
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
137
|
+
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
138
|
+
.filter(Boolean)
|
|
139
|
+
.join(", ")
|
|
140
|
+
|
|
141
|
+
const aligned = reconcileComponents(components, raw)
|
|
142
|
+
|
|
143
|
+
if (Object.keys(aligned).length <= 2) continue
|
|
144
|
+
|
|
145
|
+
const commNum = (record["Commission Number (UID)"] ?? "").trim()
|
|
146
|
+
const sourceID = commNum
|
|
147
|
+
? `${STATE_NY_NOTARIES_ADAPTER_ID}-${commNum}`
|
|
148
|
+
: stableSourceID(STATE_NY_NOTARIES_ADAPTER_ID, aligned)
|
|
149
|
+
|
|
150
|
+
yield {
|
|
151
|
+
raw,
|
|
152
|
+
components: aligned,
|
|
153
|
+
country: "US",
|
|
154
|
+
locale: "en-US",
|
|
155
|
+
source: STATE_NY_NOTARIES_ADAPTER_ID,
|
|
156
|
+
source_id: sourceID,
|
|
157
|
+
corpus_version: "",
|
|
158
|
+
license: STATE_NY_NOTARIES_DEFAULT_LICENSE,
|
|
159
|
+
}
|
|
160
|
+
emitted++
|
|
161
|
+
}
|
|
162
|
+
} finally {
|
|
163
|
+
stream.destroy()
|
|
164
|
+
}
|
|
165
|
+
},
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export const stateNyNotariesAdapter = createStateNyNotariesAdapter()
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `state-tx-notaries`: Texas Notary Public Commissions CSV consumer.
|
|
7
|
+
*
|
|
8
|
+
* The Texas Secretary of State publishes a registry of commissioned notaries public. Each row
|
|
9
|
+
* optionally carries a mailing address in free-form text (often multi-line with embedded
|
|
10
|
+
* city/state/zip). Address fill rate is ~5-10%.
|
|
11
|
+
*
|
|
12
|
+
* The adapter parses the embedded `Address` field for city/state/zip using a trailing `"CITY, ST
|
|
13
|
+
* ZIP"` pattern.
|
|
14
|
+
*
|
|
15
|
+
* License: stamped `"Public Domain"` per Texas state government open-data terms.
|
|
16
|
+
*/
|
|
17
|
+
|
|
18
|
+
import { createReadStream } from "node:fs"
|
|
19
|
+
|
|
20
|
+
import { parse as csvParse } from "csv-parse"
|
|
21
|
+
|
|
22
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
23
|
+
import { lookupStateAbbreviation } from "../../codex/us-fips-state.ts"
|
|
24
|
+
import { reconcileComponents } from "../../format.ts"
|
|
25
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
26
|
+
|
|
27
|
+
export const STATE_TX_NOTARIES_ADAPTER_ID = "state-tx-notaries"
|
|
28
|
+
export const STATE_TX_NOTARIES_DEFAULT_LICENSE = "Public Domain"
|
|
29
|
+
|
|
30
|
+
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/
|
|
31
|
+
|
|
32
|
+
/** Match trailing "CITY, ST ZIP" or "CITY, ST" at the end of an address line. */
|
|
33
|
+
const CITY_STATE_ZIP_SUFFIX = /[,]?\s*([^,]+),\s*([A-Z]{2})\s*(\d{5}(?:-\d{4})?)?\s*$/i
|
|
34
|
+
|
|
35
|
+
interface TxNotaryRow {
|
|
36
|
+
"Notary ID": string
|
|
37
|
+
"First Name": string
|
|
38
|
+
"Last Name": string
|
|
39
|
+
Address: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function splitAddress(address: string): { house_number?: string; street: string } | null {
|
|
43
|
+
const trimmed = address.trim()
|
|
44
|
+
|
|
45
|
+
if (!trimmed) return null
|
|
46
|
+
const m = HOUSE_NUMBER_PREFIX.exec(trimmed)
|
|
47
|
+
|
|
48
|
+
if (m) return { house_number: m[1], street: m[2]!.trim() }
|
|
49
|
+
|
|
50
|
+
return { street: trimmed }
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function createStateTxNotariesAdapter(): CorpusAdapter {
|
|
54
|
+
return {
|
|
55
|
+
id: STATE_TX_NOTARIES_ADAPTER_ID,
|
|
56
|
+
defaultLicense: STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
57
|
+
description:
|
|
58
|
+
"Texas Notary Public Commissions — name + mailing address with embedded city/state/zip (public-domain).",
|
|
59
|
+
|
|
60
|
+
async *rows(opts: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
61
|
+
if (opts.country && opts.country !== "US") {
|
|
62
|
+
throw new Error(`state-tx-notaries adapter: only US supported, got country=${opts.country}`)
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const stream = createReadStream(opts.inputPath, { encoding: "utf8" })
|
|
66
|
+
const parser = stream.pipe(
|
|
67
|
+
csvParse({
|
|
68
|
+
columns: true,
|
|
69
|
+
skip_empty_lines: true,
|
|
70
|
+
relax_quotes: true,
|
|
71
|
+
relax_column_count: true,
|
|
72
|
+
})
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
let emitted = 0
|
|
76
|
+
|
|
77
|
+
try {
|
|
78
|
+
for await (const record of parser as AsyncIterable<TxNotaryRow>) {
|
|
79
|
+
if (opts.signal?.aborted) break
|
|
80
|
+
|
|
81
|
+
if (opts.limit !== undefined && emitted >= opts.limit) break
|
|
82
|
+
|
|
83
|
+
const rawAddress = (record.Address ?? "").trim()
|
|
84
|
+
|
|
85
|
+
if (!rawAddress) continue
|
|
86
|
+
|
|
87
|
+
const firstName = (record["First Name"] ?? "").trim()
|
|
88
|
+
const lastName = (record["Last Name"] ?? "").trim()
|
|
89
|
+
const notaryID = (record["Notary ID"] ?? "").trim()
|
|
90
|
+
|
|
91
|
+
// Parse embedded city/state/zip from the trailing portion of the address.
|
|
92
|
+
// Addresses look like: "1215 MCMILLAN DR\nCEDAR HILL, TX 75104"
|
|
93
|
+
const addrSingleLine = rawAddress.replace(/\n/g, ", ")
|
|
94
|
+
const cszMatch = CITY_STATE_ZIP_SUFFIX.exec(addrSingleLine)
|
|
95
|
+
|
|
96
|
+
if (!cszMatch) continue
|
|
97
|
+
|
|
98
|
+
const city = (cszMatch[1] ?? "").trim()
|
|
99
|
+
const stateAbbr = (cszMatch[2] ?? "").trim()
|
|
100
|
+
const zip = (cszMatch[3] ?? "").trim()
|
|
101
|
+
|
|
102
|
+
if (!city || !stateAbbr) continue
|
|
103
|
+
|
|
104
|
+
const state = lookupStateAbbreviation(stateAbbr)
|
|
105
|
+
|
|
106
|
+
if (!state) continue
|
|
107
|
+
|
|
108
|
+
// Extract the street portion (everything before the city/state/zip)
|
|
109
|
+
const streetPortion = addrSingleLine.slice(0, cszMatch.index).replace(/,\s*$/, "").trim()
|
|
110
|
+
|
|
111
|
+
if (!streetPortion) continue
|
|
112
|
+
|
|
113
|
+
const split = splitAddress(streetPortion)
|
|
114
|
+
|
|
115
|
+
if (!split) continue
|
|
116
|
+
|
|
117
|
+
const venue = [firstName, lastName].filter(Boolean).join(" ") || undefined
|
|
118
|
+
|
|
119
|
+
const components: CanonicalRow["components"] = {
|
|
120
|
+
...(venue ? { venue } : {}),
|
|
121
|
+
...(split.house_number ? { house_number: split.house_number } : {}),
|
|
122
|
+
street: split.street,
|
|
123
|
+
locality: city,
|
|
124
|
+
region: state.abbreviation,
|
|
125
|
+
...(zip ? { postcode: zip } : {}),
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const streetPart = [split.house_number, split.street].filter(Boolean).join(" ").trim()
|
|
129
|
+
const raw = [venue, streetPart, [city, [stateAbbr, zip].filter(Boolean).join(" ")].filter(Boolean).join(", ")]
|
|
130
|
+
.filter(Boolean)
|
|
131
|
+
.join(", ")
|
|
132
|
+
|
|
133
|
+
const aligned = reconcileComponents(components, raw)
|
|
134
|
+
|
|
135
|
+
if (Object.keys(aligned).length <= 2) continue
|
|
136
|
+
|
|
137
|
+
const sourceID = notaryID
|
|
138
|
+
? `${STATE_TX_NOTARIES_ADAPTER_ID}-${notaryID}`
|
|
139
|
+
: stableSourceID(STATE_TX_NOTARIES_ADAPTER_ID, aligned)
|
|
140
|
+
|
|
141
|
+
yield {
|
|
142
|
+
raw,
|
|
143
|
+
components: aligned,
|
|
144
|
+
country: "US",
|
|
145
|
+
locale: "en-US",
|
|
146
|
+
source: STATE_TX_NOTARIES_ADAPTER_ID,
|
|
147
|
+
source_id: sourceID,
|
|
148
|
+
corpus_version: "",
|
|
149
|
+
license: STATE_TX_NOTARIES_DEFAULT_LICENSE,
|
|
150
|
+
}
|
|
151
|
+
emitted++
|
|
152
|
+
}
|
|
153
|
+
} finally {
|
|
154
|
+
stream.destroy()
|
|
155
|
+
}
|
|
156
|
+
},
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export const stateTxNotariesAdapter = createStateTxNotariesAdapter()
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `synth-po-box`: PO box / PMB / Apartado / BP synthesizer adapter.
|
|
7
|
+
*
|
|
8
|
+
* Consumes a JSONL stream of (locality, region, postcode, country) tuples — typically extracted
|
|
9
|
+
* from existing corpus output (TIGER/NAD/BAN/WOF) — and emits synthetic PO box training rows. See
|
|
10
|
+
* `../../synthesize-po-box.ts` for the per-locale templates and number-noise logic.
|
|
11
|
+
*
|
|
12
|
+
* Why an adapter and not an augmenter:
|
|
13
|
+
*
|
|
14
|
+
* - Per USPS Pub 28 / DMM 508, a PO box delivery line is mutually exclusive with a street line.
|
|
15
|
+
* Synthesizing PO boxes by mutating a street row would teach the model an invalid pattern.
|
|
16
|
+
* The clean shape is: read just (locality, region, postcode, country) and produce a fresh
|
|
17
|
+
* PO-box-shaped row.
|
|
18
|
+
* - Per-DeepSeek (3-turn consult, 2026-05-28): PMB rows that COMBINE a street line with a PMB number
|
|
19
|
+
* ARE valid (CMRA addresses). Those are produced when `pmbRatio > 0` AND the input tuple
|
|
20
|
+
* carries a `street` field.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
import { TextSpliterator } from "spliterator"
|
|
24
|
+
|
|
25
|
+
import { stableSourceID } from "../../adapter.ts"
|
|
26
|
+
import {
|
|
27
|
+
countryToLocale,
|
|
28
|
+
REGION_OPTIONAL_LOCALES,
|
|
29
|
+
synthesizeMilitaryPoBoxRow,
|
|
30
|
+
synthesizePoBoxRow,
|
|
31
|
+
type PoBoxBaseTuple,
|
|
32
|
+
} from "../../synthesize-po-box.ts"
|
|
33
|
+
import type { AdapterOptions, CanonicalRow, CorpusAdapter } from "../../types.ts"
|
|
34
|
+
|
|
35
|
+
export const SYNTH_PO_BOX_ADAPTER_ID = "synth-po-box"
|
|
36
|
+
export const SYNTH_PO_BOX_LICENSE = "Synthetic — derived from CC-BY / public-domain input tuples"
|
|
37
|
+
|
|
38
|
+
export interface PoBoxInputRow extends PoBoxBaseTuple {
|
|
39
|
+
street?: string
|
|
40
|
+
houseNumber?: string
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface SynthPoBoxAdapterOptions {
|
|
44
|
+
/**
|
|
45
|
+
* How many PO box variants to emit per input tuple. Each variant picks a different leader (and possibly a different
|
|
46
|
+
* number / noise level). Default 1.
|
|
47
|
+
*/
|
|
48
|
+
variantsPerInput?: number
|
|
49
|
+
/**
|
|
50
|
+
* Probability (0..1) of emitting a PMB-with-street variant when both the input has a street and the locale supports
|
|
51
|
+
* PMB. Default 0.15.
|
|
52
|
+
*/
|
|
53
|
+
pmbRatio?: number
|
|
54
|
+
/**
|
|
55
|
+
* Deterministic seed for reproducible synthesis. Default Date.now().
|
|
56
|
+
*/
|
|
57
|
+
seed?: number
|
|
58
|
+
/**
|
|
59
|
+
* Probability (0..1), evaluated per input tuple, of ALSO emitting one US military/diplomatic PO-box row
|
|
60
|
+
* (`PSC/CMR/Unit <id> Box <box>, APO/FPO/DPO AA/AE/AP <zip>`, #517). These rows are self-contained — they draw no
|
|
61
|
+
* field from the input tuple, so military volume scales with the input stream size. Default 0 (off) — the adapter's
|
|
62
|
+
* contract is "one row per input"; the corpus build recipe opts in to seed the rare-but-real military class without
|
|
63
|
+
* changing the default.
|
|
64
|
+
*/
|
|
65
|
+
militaryRatio?: number
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function makeRandom(seed: number): () => number {
|
|
69
|
+
let s = seed
|
|
70
|
+
|
|
71
|
+
return () => {
|
|
72
|
+
s = (s * 1664525 + 1013904223) % 4294967296
|
|
73
|
+
|
|
74
|
+
return s / 4294967296
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function createSynthPoBoxAdapter(opts: SynthPoBoxAdapterOptions = {}): CorpusAdapter {
|
|
79
|
+
const variantsPerInput = opts.variantsPerInput ?? 1
|
|
80
|
+
const pmbRatio = opts.pmbRatio ?? 0.15
|
|
81
|
+
const militaryRatio = opts.militaryRatio ?? 0
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
id: SYNTH_PO_BOX_ADAPTER_ID,
|
|
85
|
+
defaultLicense: SYNTH_PO_BOX_LICENSE,
|
|
86
|
+
description:
|
|
87
|
+
"Synthetic PO box / PMB / Apartado / Boîte Postale rows. Consumes JSONL of (locality, region, postcode, country) tuples and emits locale-appropriate PO box variants.",
|
|
88
|
+
|
|
89
|
+
async *rows(options: AdapterOptions): AsyncIterable<CanonicalRow> {
|
|
90
|
+
const random = makeRandom(opts.seed ?? Date.now())
|
|
91
|
+
|
|
92
|
+
// TextSpliterator streams string lines; the per-line try/catch below keeps this reader
|
|
93
|
+
// tolerant of malformed rows (skipped++), so TextSpliterator + explicit JSON.parse — not
|
|
94
|
+
// JSONSpliterator, which would throw on the first bad line.
|
|
95
|
+
const lines = TextSpliterator.fromAsync(options.inputPath)
|
|
96
|
+
|
|
97
|
+
let emitted = 0
|
|
98
|
+
let skipped = 0
|
|
99
|
+
let militarySeq = 0
|
|
100
|
+
|
|
101
|
+
for await (const line of lines) {
|
|
102
|
+
if (options.signal?.aborted) break
|
|
103
|
+
|
|
104
|
+
if (options.limit !== undefined && emitted >= options.limit) break
|
|
105
|
+
|
|
106
|
+
const trimmed = line.trim()
|
|
107
|
+
|
|
108
|
+
if (!trimmed) continue
|
|
109
|
+
|
|
110
|
+
let input: PoBoxInputRow
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
input = JSON.parse(trimmed) as PoBoxInputRow
|
|
114
|
+
} catch {
|
|
115
|
+
skipped++
|
|
116
|
+
continue
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Region is required EXCEPT for region-less locales (NZ: `Private Bag 12, Auckland 1010`
|
|
120
|
+
// has no region token, #517). synthesizePoBoxRow handles region absence; the guard just
|
|
121
|
+
// must not discard those tuples as "missing region".
|
|
122
|
+
const regionOptional = input.country ? REGION_OPTIONAL_LOCALES.has(countryToLocale(input.country)) : false
|
|
123
|
+
|
|
124
|
+
if (!input.locality || !input.postcode || !input.country || (!input.region && !regionOptional)) {
|
|
125
|
+
skipped++
|
|
126
|
+
continue
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
if (options.country && options.country !== input.country) continue
|
|
130
|
+
|
|
131
|
+
for (let v = 0; v < variantsPerInput; v++) {
|
|
132
|
+
const synth = synthesizePoBoxRow(input, { random, pmbRatio })
|
|
133
|
+
|
|
134
|
+
if (!synth) continue
|
|
135
|
+
|
|
136
|
+
// Include `v` in dependent_locality slot to vary the digest across variants;
|
|
137
|
+
// stableSourceID only accepts ComponentTag keys.
|
|
138
|
+
const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
|
|
139
|
+
locality: `${input.locality}#${v}`,
|
|
140
|
+
region: input.region,
|
|
141
|
+
postcode: input.postcode,
|
|
142
|
+
country: input.country,
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
yield {
|
|
146
|
+
raw: synth.raw,
|
|
147
|
+
components: synth.components,
|
|
148
|
+
country: input.country,
|
|
149
|
+
locale: synth.locale,
|
|
150
|
+
source: SYNTH_PO_BOX_ADAPTER_ID,
|
|
151
|
+
source_id: sourceID,
|
|
152
|
+
corpus_version: "",
|
|
153
|
+
license: SYNTH_PO_BOX_LICENSE,
|
|
154
|
+
}
|
|
155
|
+
emitted++
|
|
156
|
+
|
|
157
|
+
if (options.limit !== undefined && emitted >= options.limit) break
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// US military/diplomatic PO-box rows (#517): self-contained — draw nothing from the input
|
|
161
|
+
// tuple — emitted per input line with probability `militaryRatio` (off by default, so the
|
|
162
|
+
// default random stream and output are byte-identical). Military volume scales with the
|
|
163
|
+
// stream rather than the US-tuple count. US-only: suppressed under a non-US country filter
|
|
164
|
+
// and counted against `limit` like any other row.
|
|
165
|
+
const militaryAllowed = !options.country || options.country === "US"
|
|
166
|
+
|
|
167
|
+
if (
|
|
168
|
+
militaryRatio > 0 &&
|
|
169
|
+
militaryAllowed &&
|
|
170
|
+
(options.limit === undefined || emitted < options.limit) &&
|
|
171
|
+
random() < militaryRatio
|
|
172
|
+
) {
|
|
173
|
+
const mil = synthesizeMilitaryPoBoxRow({ random })
|
|
174
|
+
const sourceID = stableSourceID(SYNTH_PO_BOX_ADAPTER_ID, {
|
|
175
|
+
po_box: `${mil.components.po_box}#mil${militarySeq++}`,
|
|
176
|
+
locality: mil.components.locality!,
|
|
177
|
+
region: mil.components.region!,
|
|
178
|
+
postcode: mil.components.postcode!,
|
|
179
|
+
})
|
|
180
|
+
yield {
|
|
181
|
+
raw: mil.raw,
|
|
182
|
+
components: mil.components,
|
|
183
|
+
country: "US",
|
|
184
|
+
locale: mil.locale,
|
|
185
|
+
source: SYNTH_PO_BOX_ADAPTER_ID,
|
|
186
|
+
source_id: sourceID,
|
|
187
|
+
corpus_version: "",
|
|
188
|
+
license: SYNTH_PO_BOX_LICENSE,
|
|
189
|
+
}
|
|
190
|
+
emitted++
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
},
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export const synthPoBoxAdapter = createSynthPoBoxAdapter()
|