@mailwoman/corpus 8.5.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/out/src/adapter.d.ts +41 -0
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +46 -1
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +4 -3
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +60 -67
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +0 -6
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -24
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +70 -76
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +44 -49
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +4 -7
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +6 -11
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -7
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +3 -7
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +55 -73
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +58 -76
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +67 -85
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +69 -87
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +7 -15
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +62 -80
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +60 -82
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +58 -65
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +5 -8
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +58 -76
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +51 -71
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +2 -1
- package/out/src/build.js.map +1 -1
- package/out/src/golden.d.ts +5 -0
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +23 -9
- package/out/src/golden.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +2 -13
- package/out/src/parquet.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +2 -1
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +68 -60
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +14 -58
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +11 -58
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +2 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +24 -62
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +14 -16
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +51 -94
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +59 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +51 -30
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +56 -82
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +231 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +669 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +22 -64
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +7 -0
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +7 -0
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +1 -12
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -7
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +2 -1
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +513 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +5 -4
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +8 -6
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/wof-json.d.ts +2 -2
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +5 -8
- package/out/src/wof-json.js.map +1 -1
- package/package.json +8 -8
- package/src/adapter.ts +58 -1
- package/src/adapters/ban/adapter.ts +55 -65
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +2 -33
- package/src/adapters/geonames/adapter.ts +64 -72
- package/src/adapters/geonames-postal/adapter.ts +42 -50
- package/src/adapters/gnaf/adapter.ts +4 -7
- package/src/adapters/gnaf/assemble.ts +6 -10
- package/src/adapters/openaddresses/adapter.ts +2 -7
- package/src/adapters/overture/adapter.ts +3 -6
- package/src/adapters/state-hi-schools/adapter.ts +50 -73
- package/src/adapters/state-ia-contractors/adapter.ts +52 -76
- package/src/adapters/state-ny-notaries/adapter.ts +61 -85
- package/src/adapters/state-tx-notaries/adapter.ts +60 -84
- package/src/adapters/synth-po-box/adapter.ts +7 -17
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +61 -84
- package/src/adapters/usgov-imls-pls/adapter.ts +54 -82
- package/src/adapters/usgov-irs-bmf/adapter.ts +53 -63
- package/src/adapters/usgov-nad/adapter.ts +5 -8
- package/src/adapters/usgov-nppes/adapter.ts +51 -75
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +45 -71
- package/src/adapters/wof-admin-jp/adapter.ts +34 -8
- package/src/build.ts +2 -1
- package/src/golden.ts +24 -9
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/parquet.ts +3 -16
- package/src/shard-recipes/anchor-absorption.ts +2 -1
- package/src/shard-recipes/country-balanced.ts +69 -70
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-order.ts +13 -63
- package/src/shard-recipes/german.ts +12 -65
- package/src/shard-recipes/index.ts +2 -0
- package/src/shard-recipes/intersection.ts +32 -60
- package/src/shard-recipes/locale.ts +14 -16
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/po-box-cedex.ts +68 -119
- package/src/shard-recipes/scaffold.ts +82 -28
- package/src/shard-recipes/street-affix.ts +58 -95
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +22 -70
- package/src/split.ts +7 -0
- package/src/synthesize.ts +1 -15
- package/src/tools/align-shard.ts +3 -6
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +2 -1
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +748 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +12 -7
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/wof-json.ts +5 -8
package/README.md
CHANGED
|
@@ -72,7 +72,7 @@ node scripts/corpus-stats.mjs
|
|
|
72
72
|
- [`@mailwoman/neural-weights-en-us`](../neural-weights-en-us) — the trained model itself
|
|
73
73
|
- [Corpus Construction concepts](https://mailwoman.sister.software/articles/concepts/corpus-construction/)
|
|
74
74
|
- [Training Pipeline concepts](https://mailwoman.sister.software/articles/concepts/training-pipeline/)
|
|
75
|
-
- [CONTRIBUTING_MODEL_WORK](https://
|
|
75
|
+
- [CONTRIBUTING_MODEL_WORK](https://github.com/sister-software/mailwoman/blob/main/docs/engineering/CONTRIBUTING_MODEL_WORK.mdx)
|
|
76
76
|
|
|
77
77
|
## License
|
|
78
78
|
|
package/out/src/adapter.d.ts
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* - `InMemoryAdapterRegistry`: the default implementation.
|
|
13
13
|
* - `stableSourceID(adapterID, components)`: deterministic content-addressed id for adapters whose
|
|
14
14
|
* source data has no native primary key (CSV, GeoJSON).
|
|
15
|
+
* - `HOUSE_NUMBER_PREFIX` + `splitStreetLine(line)`: the one house-number/street split every
|
|
16
|
+
* US CSV adapter uses.
|
|
15
17
|
* - `canonicalDedupKey(row)`: normalized signature used to drop near-identical rows during a run.
|
|
16
18
|
* Adapter-internal dedup; cross-adapter dedup is the runner's job.
|
|
17
19
|
* - `streamingSha256()`: thin wrapper around `node:crypto` so the runner can hash JSONL output as it
|
|
@@ -74,6 +76,40 @@ export declare const defaultAdapterRegistry: InMemoryAdapterRegistry;
|
|
|
74
76
|
* length.
|
|
75
77
|
*/
|
|
76
78
|
export declare function stableSourceID(adapterID: string, components: Partial<Record<ComponentTag, string>>): string;
|
|
79
|
+
/**
|
|
80
|
+
* Leading house number on a US-style street line: digits, an optional hyphenated range (Queens `40-12`), an optional
|
|
81
|
+
* single alpha suffix (`101A`), then whitespace, then the rest.
|
|
82
|
+
*
|
|
83
|
+
* This decides where the house number ends and the street begins for every US CSV adapter, so an edge case fixed here
|
|
84
|
+
* is fixed for all of them.
|
|
85
|
+
*
|
|
86
|
+
* The remainder must stay `(\S.*)` and must NOT become `(.+)`: `\s+` and `.` both match a tab, so `\s+(.+)$` lets the
|
|
87
|
+
* engine split a run of tabs between the two groups every possible way before failing — quadratic backtracking on
|
|
88
|
+
* attacker-shaped input. Requiring a non-space start removes the overlap. Group 2 is trimmed by the caller either way,
|
|
89
|
+
* so the two forms are indistinguishable on real input; only the failure cost differs.
|
|
90
|
+
*/
|
|
91
|
+
export declare const HOUSE_NUMBER_PREFIX: RegExp;
|
|
92
|
+
/**
|
|
93
|
+
* A street line split into its house number and the remainder.
|
|
94
|
+
*/
|
|
95
|
+
export interface SplitStreetLine {
|
|
96
|
+
house_number?: string;
|
|
97
|
+
street: string;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Split a US-style street line on {@link HOUSE_NUMBER_PREFIX}.
|
|
101
|
+
*
|
|
102
|
+
* The US CSV sources follow USPS Publication 28 conventions with hand-entry drift. The leading digit run is the house
|
|
103
|
+
* number (`"123 Main St"`, `"6450 W Indian School Rd"`), and the regex tolerates one trailing letter (`"123A Main St"`)
|
|
104
|
+
* plus an optional hyphenated half (`"40-12 Bell Blvd"`, common in NYC and suburban garden-apartment numbering; Hawaii
|
|
105
|
+
* uses it island-wide — `"47-470 Hui Aeko Place"`).
|
|
106
|
+
*
|
|
107
|
+
* Returns `null` for blank input. Anything that does not match the prefix shape (`"PO Box 1234"`, `"RR 2 Box 67"`, `"HC
|
|
108
|
+
* 1"`) becomes a single `street` value rather than being mangled — the model sees the original surface form and
|
|
109
|
+
* downstream classifiers pick it up. Callers that need those forms recognized as something other than a street (see
|
|
110
|
+
* `usgov-irs-bmf`) test for them BEFORE calling this.
|
|
111
|
+
*/
|
|
112
|
+
export declare function splitStreetLine(line: string): SplitStreetLine | null;
|
|
77
113
|
/**
|
|
78
114
|
* Canonical dedup key for a row.
|
|
79
115
|
*
|
|
@@ -99,5 +135,10 @@ export interface StreamingHasher {
|
|
|
99
135
|
/**
|
|
100
136
|
* Default `StreamingHasher` (SHA-256, hex).
|
|
101
137
|
*/
|
|
138
|
+
/**
|
|
139
|
+
* @see {@link StreamingHasher} — this is the incremental counterpart to `sha256Hex`, not a copy of
|
|
140
|
+
* it. `@mailwoman/core/utils` exposes one-shot digests (`sha256Hex`) and whole-file digests
|
|
141
|
+
* (`sha256File`); the runner needs neither, because it hashes JSONL lines as they stream past.
|
|
142
|
+
*/
|
|
102
143
|
export declare function streamingSha256(): StreamingHasher;
|
|
103
144
|
//# sourceMappingURL=adapter.d.ts.map
|
package/out/src/adapter.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../src/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAIH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAGzD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,YAAY,CAAA;AAE7D;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI,CAAA;IAEtC;;OAEG;IACH,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAAA;IAE1C;;OAEG;IACH,IAAI,IAAI,SAAS,aAAa,EAAE,CAAA;IAEhC;;OAEG;IACH,GAAG,IAAI,SAAS,MAAM,EAAE,CAAA;CACxB;AAED;;;GAGG;AACH,qBAAa,uBAAwB,YAAW,eAAe;;IAG9D,QAAQ,CAAC,OAAO,EAAE,aAAa,GAAG,IAAI;IAQtC,GAAG,CAAC,EAAE,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS;IAI1C,IAAI,IAAI,SAAS,aAAa,EAAE;IAIhC,GAAG,IAAI,SAAS,MAAM,EAAE;CAGxB;AAED;;;GAGG;AACH,eAAO,MAAM,sBAAsB,yBAAgC,CAAA;AAEnE;;;;;;;;;;GAUG;AACH,wBAAgB,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,GAAG,MAAM,CAQ3G;AAED;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,mBAAmB,QAAuC,CAAA;AAEvE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,YAAY,CAAC,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAUpE;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,CAAC,GAAG,EAAE,YAAY,GAAG,MAAM,CAO3D;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC/B,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,UAAU,GAAG,IAAI,CAAA;IACxC,MAAM,IAAI,MAAM,CAAA;CAChB;AAED;;GAEG;AACH;;;;GAIG;AACH,wBAAgB,eAAe,IAAI,eAAe,CAmBjD"}
|
package/out/src/adapter.js
CHANGED
|
@@ -12,6 +12,8 @@
|
|
|
12
12
|
* - `InMemoryAdapterRegistry`: the default implementation.
|
|
13
13
|
* - `stableSourceID(adapterID, components)`: deterministic content-addressed id for adapters whose
|
|
14
14
|
* source data has no native primary key (CSV, GeoJSON).
|
|
15
|
+
* - `HOUSE_NUMBER_PREFIX` + `splitStreetLine(line)`: the one house-number/street split every
|
|
16
|
+
* US CSV adapter uses.
|
|
15
17
|
* - `canonicalDedupKey(row)`: normalized signature used to drop near-identical rows during a run.
|
|
16
18
|
* Adapter-internal dedup; cross-adapter dedup is the runner's job.
|
|
17
19
|
* - `streamingSha256()`: thin wrapper around `node:crypto` so the runner can hash JSONL output as it
|
|
@@ -20,6 +22,7 @@
|
|
|
20
22
|
* Everything here is pure (no I/O); side-effecting code goes in `./runner.ts`.
|
|
21
23
|
*/
|
|
22
24
|
import { createHash } from "node:crypto";
|
|
25
|
+
import { sha256Hex } from "@mailwoman/core/utils";
|
|
23
26
|
/**
|
|
24
27
|
* Default in-memory registry. The runner constructs one per invocation; the CLI re-uses a shared singleton
|
|
25
28
|
* (`defaultAdapterRegistry`) populated by `./adapters/index.ts` as adapters come online.
|
|
@@ -61,9 +64,46 @@ export const defaultAdapterRegistry = new InMemoryAdapterRegistry();
|
|
|
61
64
|
export function stableSourceID(adapterID, components) {
|
|
62
65
|
const sortedKeys = Object.keys(components).toSorted();
|
|
63
66
|
const payload = sortedKeys.map((k) => `${k}=${components[k] ?? ""}`).join("\u001F");
|
|
64
|
-
|
|
67
|
+
// One `update` over the joined string, not three chained ones — identical byte stream either way,
|
|
68
|
+
// but the separator has to stay inline or every id in every shard changes.
|
|
69
|
+
const digest = sha256Hex(`${adapterID}\u001E${payload}`);
|
|
65
70
|
return `${adapterID}-${digest.slice(0, 12)}`;
|
|
66
71
|
}
|
|
72
|
+
/**
|
|
73
|
+
* Leading house number on a US-style street line: digits, an optional hyphenated range (Queens `40-12`), an optional
|
|
74
|
+
* single alpha suffix (`101A`), then whitespace, then the rest.
|
|
75
|
+
*
|
|
76
|
+
* This decides where the house number ends and the street begins for every US CSV adapter, so an edge case fixed here
|
|
77
|
+
* is fixed for all of them.
|
|
78
|
+
*
|
|
79
|
+
* The remainder must stay `(\S.*)` and must NOT become `(.+)`: `\s+` and `.` both match a tab, so `\s+(.+)$` lets the
|
|
80
|
+
* engine split a run of tabs between the two groups every possible way before failing — quadratic backtracking on
|
|
81
|
+
* attacker-shaped input. Requiring a non-space start removes the overlap. Group 2 is trimmed by the caller either way,
|
|
82
|
+
* so the two forms are indistinguishable on real input; only the failure cost differs.
|
|
83
|
+
*/
|
|
84
|
+
export const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(\S.*)$/;
|
|
85
|
+
/**
|
|
86
|
+
* Split a US-style street line on {@link HOUSE_NUMBER_PREFIX}.
|
|
87
|
+
*
|
|
88
|
+
* The US CSV sources follow USPS Publication 28 conventions with hand-entry drift. The leading digit run is the house
|
|
89
|
+
* number (`"123 Main St"`, `"6450 W Indian School Rd"`), and the regex tolerates one trailing letter (`"123A Main St"`)
|
|
90
|
+
* plus an optional hyphenated half (`"40-12 Bell Blvd"`, common in NYC and suburban garden-apartment numbering; Hawaii
|
|
91
|
+
* uses it island-wide — `"47-470 Hui Aeko Place"`).
|
|
92
|
+
*
|
|
93
|
+
* Returns `null` for blank input. Anything that does not match the prefix shape (`"PO Box 1234"`, `"RR 2 Box 67"`, `"HC
|
|
94
|
+
* 1"`) becomes a single `street` value rather than being mangled — the model sees the original surface form and
|
|
95
|
+
* downstream classifiers pick it up. Callers that need those forms recognized as something other than a street (see
|
|
96
|
+
* `usgov-irs-bmf`) test for them BEFORE calling this.
|
|
97
|
+
*/
|
|
98
|
+
export function splitStreetLine(line) {
|
|
99
|
+
const trimmed = line.trim();
|
|
100
|
+
if (!trimmed)
|
|
101
|
+
return null;
|
|
102
|
+
const match = HOUSE_NUMBER_PREFIX.exec(trimmed);
|
|
103
|
+
if (match)
|
|
104
|
+
return { house_number: match[1], street: match[2].trim() };
|
|
105
|
+
return { street: trimmed };
|
|
106
|
+
}
|
|
67
107
|
/**
|
|
68
108
|
* Canonical dedup key for a row.
|
|
69
109
|
*
|
|
@@ -84,6 +124,11 @@ export function canonicalDedupKey(row) {
|
|
|
84
124
|
/**
|
|
85
125
|
* Default `StreamingHasher` (SHA-256, hex).
|
|
86
126
|
*/
|
|
127
|
+
/**
|
|
128
|
+
* @see {@link StreamingHasher} — this is the incremental counterpart to `sha256Hex`, not a copy of
|
|
129
|
+
* it. `@mailwoman/core/utils` exposes one-shot digests (`sha256Hex`) and whole-file digests
|
|
130
|
+
* (`sha256File`); the runner needs neither, because it hashes JSONL lines as they stream past.
|
|
131
|
+
*/
|
|
87
132
|
export function streamingSha256() {
|
|
88
133
|
const h = createHash("sha256");
|
|
89
134
|
let finalized = false;
|
package/out/src/adapter.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../src/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../src/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAa,MAAM,aAAa,CAAA;AAGnD,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAA;AAiCjD;;;GAGG;AACH,MAAM,OAAO,uBAAuB;IACnC,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAA;IAExC,QAAQ,CAAC,OAAsB;QAC9B,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,uBAAuB,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,CAAC,qBAAqB,CAAC,CAAA;QACxF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAA;IACpC,CAAC;IAED,GAAG,CAAC,EAAU;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAC1B,CAAC;IAED,IAAI;QACH,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,GAAG;QACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACrC,CAAC;CACD;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,IAAI,uBAAuB,EAAE,CAAA;AAEnE;;;;;;;;;;GAUG;AACH,MAAM,UAAU,cAAc,CAAC,SAAiB,EAAE,UAAiD;IAClG,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAoB,CAAA;IACvE,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnF,kGAAkG;IAClG,2EAA2E;IAC3E,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,SAAS,SAAS,OAAO,EAAE,CAAC,CAAA;IAExD,OAAO,GAAG,SAAS,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAA;AAC7C,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,oCAAoC,CAAA;AAUvE;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;IAE3B,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,MAAM,KAAK,GAAG,mBAAmB,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IAE/C,IAAI,KAAK;QAAE,OAAO,EAAE,YAAY,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,EAAE,CAAA;IAEtE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AAC3B,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,iBAAiB,CAAC,GAAiB;IAClD,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,QAAQ,EAAoB,CAAA;IAC3E,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACxF,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IACpE,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IAE9D,OAAO,GAAG,GAAG,CAAC,OAAO,SAAS,OAAO,SAAS,QAAQ,GAAG,SAAS,EAAE,CAAA;AACrE,CAAC;AAcD;;GAEG;AACH;;;;GAIG;AACH,MAAM,UAAU,eAAe;IAC9B,MAAM,CAAC,GAAS,UAAU,CAAC,QAAQ,CAAC,CAAA;IACpC,IAAI,SAAS,GAAG,KAAK,CAAA;IACrB,IAAI,SAAS,GAAG,EAAE,CAAA;IAElB,OAAO;QACN,MAAM,CAAC,KAAK;YACX,IAAI,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAA;YACjF,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QACpD,CAAC;QACD,MAAM;YACL,IAAI,CAAC,SAAS,EAAE,CAAC;gBAChB,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;gBAC3B,SAAS,GAAG,IAAI,CAAA;YACjB,CAAC;YAED,OAAO,SAAS,CAAA;QACjB,CAAC;KACD,CAAA;AACF,CAAC"}
|
|
@@ -25,9 +25,10 @@
|
|
|
25
25
|
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
26
|
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
27
27
|
*
|
|
28
|
-
* The adapter is streaming-aware:
|
|
29
|
-
* sits in memory. Honors `opts.limit` for fixture / smoke runs,
|
|
30
|
-
* and `opts.country` for a self-consistency check (errors if
|
|
28
|
+
* The adapter is streaming-aware: `CSVSpliterator.fromAsync` reads the `;`-delimited dump row by
|
|
29
|
+
* row, so a 25M-row file never sits in memory. Honors `opts.limit` for fixture / smoke runs,
|
|
30
|
+
* `opts.signal` for cancellation, and `opts.country` for a self-consistency check (errors if
|
|
31
|
+
* country !== FR).
|
|
31
32
|
*/
|
|
32
33
|
import type { CorpusAdapter } from "../../types.ts";
|
|
33
34
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAMH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAGjF;;;GAGG;AACH,eAAO,MAAM,cAAc,QAAQ,CAAA;AAsDnC,wBAAgB,gBAAgB,IAAI,aAAa,CAqFhD;AAED;;GAEG;AACH,eAAO,MAAM,UAAU,eAAqB,CAAA"}
|
|
@@ -25,12 +25,12 @@
|
|
|
25
25
|
* conservative `ODbL-1.0` label, which wrongly read as Tier-C-denied in the corpus license audit.
|
|
26
26
|
* The model card MUST carry the BAN attribution (Tier B obligation).
|
|
27
27
|
*
|
|
28
|
-
* The adapter is streaming-aware:
|
|
29
|
-
* sits in memory. Honors `opts.limit` for fixture / smoke runs,
|
|
30
|
-
* and `opts.country` for a self-consistency check (errors if
|
|
28
|
+
* The adapter is streaming-aware: `CSVSpliterator.fromAsync` reads the `;`-delimited dump row by
|
|
29
|
+
* row, so a 25M-row file never sits in memory. Honors `opts.limit` for fixture / smoke runs,
|
|
30
|
+
* `opts.signal` for cancellation, and `opts.country` for a self-consistency check (errors if
|
|
31
|
+
* country !== FR).
|
|
31
32
|
*/
|
|
32
|
-
import {
|
|
33
|
-
import { parse as csvParse } from "csv-parse";
|
|
33
|
+
import { CSVSpliterator } from "spliterator";
|
|
34
34
|
import { stableSourceID } from "../../adapter.js";
|
|
35
35
|
import { reconcileComponents } from "../../format.js";
|
|
36
36
|
import { decomposeFrStreet } from "./street-decompose.js";
|
|
@@ -80,70 +80,63 @@ export function createBanAdapter() {
|
|
|
80
80
|
if (opts.country && opts.country !== "FR") {
|
|
81
81
|
throw new Error(`ban adapter: only FR supported, got country=${opts.country}`);
|
|
82
82
|
}
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
relax_column_count: true,
|
|
90
|
-
}));
|
|
83
|
+
const rows = CSVSpliterator.fromAsync(opts.inputPath, {
|
|
84
|
+
mode: "object",
|
|
85
|
+
normalizeKeys: false,
|
|
86
|
+
columnDelimiter: ";",
|
|
87
|
+
enableQuoteHandling: true,
|
|
88
|
+
});
|
|
91
89
|
let emitted = 0;
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
components.house_number = house;
|
|
110
|
-
}
|
|
111
|
-
if (decomposed.prefix) {
|
|
112
|
-
components.street_prefix = decomposed.prefix;
|
|
113
|
-
}
|
|
114
|
-
if (decomposed.street) {
|
|
115
|
-
components.street = decomposed.street;
|
|
116
|
-
}
|
|
117
|
-
if (postcode) {
|
|
118
|
-
components.postcode = postcode;
|
|
119
|
-
}
|
|
120
|
-
if (locality) {
|
|
121
|
-
components.locality = locality;
|
|
122
|
-
}
|
|
123
|
-
const raw = composeRaw(house, street, postcode, locality);
|
|
124
|
-
if (!raw)
|
|
125
|
-
continue;
|
|
126
|
-
const aligned = reconcileComponents(components, raw);
|
|
127
|
-
if (!Object.keys(aligned).length)
|
|
128
|
-
continue;
|
|
129
|
-
const sourceID = record.id?.trim()
|
|
130
|
-
? `${BAN_ADAPTER_ID}-${record.id.trim()}`
|
|
131
|
-
: stableSourceID(BAN_ADAPTER_ID, aligned);
|
|
132
|
-
yield {
|
|
133
|
-
raw,
|
|
134
|
-
components: aligned,
|
|
135
|
-
country: "FR",
|
|
136
|
-
locale: "fr-FR",
|
|
137
|
-
source: BAN_ADAPTER_ID,
|
|
138
|
-
source_id: sourceID,
|
|
139
|
-
corpus_version: "",
|
|
140
|
-
license: "Licence Ouverte 2.0",
|
|
141
|
-
};
|
|
142
|
-
emitted++;
|
|
90
|
+
for await (const record of rows) {
|
|
91
|
+
if (opts.signal?.aborted)
|
|
92
|
+
break;
|
|
93
|
+
if (opts.limit !== undefined && emitted >= opts.limit)
|
|
94
|
+
break;
|
|
95
|
+
const house = composeHouseNumber(record.numero ?? "", record.rep ?? "");
|
|
96
|
+
const street = (record.nom_voie ?? "").trim();
|
|
97
|
+
const postcode = (record.code_postal ?? "").trim();
|
|
98
|
+
const locality = (record.nom_commune ?? "").trim();
|
|
99
|
+
if (!street || !locality)
|
|
100
|
+
continue;
|
|
101
|
+
if (!house && !postcode)
|
|
102
|
+
continue;
|
|
103
|
+
const decomposed = decomposeFrStreet(street);
|
|
104
|
+
const components = {};
|
|
105
|
+
if (house) {
|
|
106
|
+
components.house_number = house;
|
|
143
107
|
}
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
108
|
+
if (decomposed.prefix) {
|
|
109
|
+
components.street_prefix = decomposed.prefix;
|
|
110
|
+
}
|
|
111
|
+
if (decomposed.street) {
|
|
112
|
+
components.street = decomposed.street;
|
|
113
|
+
}
|
|
114
|
+
if (postcode) {
|
|
115
|
+
components.postcode = postcode;
|
|
116
|
+
}
|
|
117
|
+
if (locality) {
|
|
118
|
+
components.locality = locality;
|
|
119
|
+
}
|
|
120
|
+
const raw = composeRaw(house, street, postcode, locality);
|
|
121
|
+
if (!raw)
|
|
122
|
+
continue;
|
|
123
|
+
const aligned = reconcileComponents(components, raw);
|
|
124
|
+
if (!Object.keys(aligned).length)
|
|
125
|
+
continue;
|
|
126
|
+
const sourceID = record.id?.trim()
|
|
127
|
+
? `${BAN_ADAPTER_ID}-${record.id.trim()}`
|
|
128
|
+
: stableSourceID(BAN_ADAPTER_ID, aligned);
|
|
129
|
+
yield {
|
|
130
|
+
raw,
|
|
131
|
+
components: aligned,
|
|
132
|
+
country: "FR",
|
|
133
|
+
locale: "fr-FR",
|
|
134
|
+
source: BAN_ADAPTER_ID,
|
|
135
|
+
source_id: sourceID,
|
|
136
|
+
corpus_version: "",
|
|
137
|
+
license: "Licence Ouverte 2.0",
|
|
138
|
+
};
|
|
139
|
+
emitted++;
|
|
147
140
|
}
|
|
148
141
|
},
|
|
149
142
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/ban/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAErD,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAA;AAEzD;;;GAGG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,KAAK,CAAA;AAenC;;;GAGG;AACH,SAAS,kBAAkB,CAAC,MAAc,EAAE,GAAW;IACtD,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IACvB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAEpB,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IAEjB,OAAO,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,UAAU,CAAC,KAAa,EAAE,MAAc,EAAE,QAAgB,EAAE,QAAgB;IACpF,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEnE,IAAI,UAAU,EAAE,CAAC;QAChB,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IACvB,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEtE,IAAI,QAAQ,EAAE,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACvD,CAAC;AAED,MAAM,UAAU,gBAAgB;IAC/B,OAAO;QACN,EAAE,EAAE,cAAc;QAClB,cAAc,EAAE,qBAAqB;QACrC,WAAW,EAAE,+EAA+E;QAE5F,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,+CAA+C,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YAC/E,CAAC;YAED,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE;gBACrD,IAAI,EAAE,QAAQ;gBACd,aAAa,EAAE,KAAK;gBACpB,eAAe,EAAE,GAAG;gBACpB,mBAAmB,EAAE,IAAI;aACzB,CAAC,CAAA;YAEF,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,KAAK,EAAE,MAAM,MAAM,IAAI,IAA6B,EAAE,CAAC;gBAC1D,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,MAAK;gBAE/B,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;oBAAE,MAAK;gBAE5D,MAAM,KAAK,GAAG,kBAAkB,CAAC,MAAM,CAAC,MAAM,IAAI,EAAE,EAAE,MAAM,CAAC,GAAG,IAAI,EAAE,CAAC,CAAA;gBACvE,MAAM,MAAM,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAC7C,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAClD,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;gBAElD,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBAElC,IAAI,CAAC,KAAK,IAAI,CAAC,QAAQ;oBAAE,SAAQ;gBAEjC,MAAM,UAAU,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;gBAE5C,MAAM,UAAU,GAA+B,EAAE,CAAA;gBAEjD,IAAI,KAAK,EAAE,CAAC;oBACX,UAAU,CAAC,YAAY,GAAG,KAAK,CAAA;gBAChC,CAAC;gBAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACvB,UAAU,CAAC,aAAa,GAAG,UAAU,CAAC,MAAM,CAAA;gBAC7C,CAAC;gBAED,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;oBACvB,UAAU,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAA;gBACtC,CAAC;gBAED,IAAI,QAAQ,EAAE,CAAC;oBACd,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;gBAC/B,CAAC;gBAED,IAAI,QAAQ,EAAE,CAAC;oBACd,UAAU,CAAC,QAAQ,GAAG,QAAQ,CAAA;gBAC/B,CAAC;gBAED,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAA;gBAEzD,IAAI,CAAC,GAAG;oBAAE,SAAQ;gBAElB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAEpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;oBAAE,SAAQ;gBAE1C,MAAM,QAAQ,GAAG,MAAM,CAAC,EAAE,EAAE,IAAI,EAAE;oBACjC,CAAC,CAAC,GAAG,cAAc,IAAI,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE;oBACzC,CAAC,CAAC,cAAc,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;gBAE1C,MAAM;oBACL,GAAG;oBACH,UAAU,EAAE,OAAO;oBACnB,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,OAAO;oBACf,MAAM,EAAE,cAAc;oBACtB,SAAS,EAAE,QAAQ;oBACnB,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,qBAAqB;iBAC9B,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAG,gBAAgB,EAAE,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"street-decompose.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/street-decompose.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;
|
|
1
|
+
{"version":3,"file":"street-decompose.d.ts","sourceRoot":"","sources":["../../../../src/adapters/ban/street-decompose.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAyCH,MAAM,WAAW,kBAAkB;IAClC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,MAAM,GAAG,kBAAkB,CA4BtE"}
|
|
@@ -15,40 +15,33 @@
|
|
|
15
15
|
* Sources street types from `core/data/libpostal/dictionaries/fr/street_types.txt`.
|
|
16
16
|
*/
|
|
17
17
|
import { readFileSync } from "node:fs";
|
|
18
|
-
import { resolve } from "node:path";
|
|
19
18
|
import { resourceDictionaryPath } from "@mailwoman/core/utils";
|
|
19
|
+
import { TextSpliterator } from "spliterator";
|
|
20
20
|
/**
|
|
21
21
|
* Tokens a BAN street needs before a type/article/name decomposition is attempted.
|
|
22
22
|
*/
|
|
23
23
|
const MIN_TOKENS_FOR_DECOMPOSE = 3;
|
|
24
24
|
function loadDictionary(filename) {
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
set.add(f);
|
|
42
|
-
}
|
|
43
|
-
}
|
|
25
|
+
// `resourceDictionaryPath` already resolves both layouts — `core/data/...` from source and from the
|
|
26
|
+
// packaged `out/` tree. The candidate list this replaced named it TWICE and then guessed a third path
|
|
27
|
+
// off `process.cwd()`, and swallowed every error while probing, so a corrupt dictionary reported as a
|
|
28
|
+
// missing one.
|
|
29
|
+
const text = readFileSync(resourceDictionaryPath("libpostal", "fr", filename), "utf8");
|
|
30
|
+
const set = new Set();
|
|
31
|
+
// The largest libpostal dictionary is 8.4 KB, and this runs once per process at module load.
|
|
32
|
+
for (const line of TextSpliterator.from(text)) {
|
|
33
|
+
const trimmed = line.trim();
|
|
34
|
+
if (!trimmed || trimmed.startsWith("#"))
|
|
35
|
+
continue;
|
|
36
|
+
// libpostal format: canonical|abbr|abbr|... — index all forms
|
|
37
|
+
for (const form of trimmed.split("|")) {
|
|
38
|
+
const f = form.trim().toLowerCase();
|
|
39
|
+
if (f) {
|
|
40
|
+
set.add(f);
|
|
44
41
|
}
|
|
45
|
-
return set;
|
|
46
|
-
}
|
|
47
|
-
catch {
|
|
48
|
-
// try next
|
|
49
42
|
}
|
|
50
43
|
}
|
|
51
|
-
|
|
44
|
+
return set;
|
|
52
45
|
}
|
|
53
46
|
const STREET_TYPES_FR = loadDictionary("street_types.txt");
|
|
54
47
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"street-decompose.js","sourceRoot":"","sources":["../../../../src/adapters/ban/street-decompose.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;
|
|
1
|
+
{"version":3,"file":"street-decompose.js","sourceRoot":"","sources":["../../../../src/adapters/ban/street-decompose.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,MAAM,wBAAwB,GAAG,CAAC,CAAA;AAElC,SAAS,cAAc,CAAC,QAAgB;IACvC,oGAAoG;IACpG,sGAAsG;IACtG,sGAAsG;IACtG,eAAe;IACf,MAAM,IAAI,GAAG,YAAY,CAAC,sBAAsB,CAAC,WAAW,EAAE,IAAI,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAA;IACtF,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAA;IAE7B,6FAA6F;IAC7F,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAQ;QAEjD,8DAA8D;QAC9D,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAEnC,IAAI,CAAC,EAAE,CAAC;gBACP,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YACX,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,MAAM,eAAe,GAAG,cAAc,CAAC,kBAAkB,CAAC,CAAA;AAO1D;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,QAAgB;IACjD,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAA;IAE/B,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,CAAA;IAEjD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IAEnC,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;IAE/D,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAA;IAEjE,iDAAiD;IACjD,IAAI,MAAM,CAAC,MAAM,IAAI,wBAAwB,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAA;QAEzD,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAClC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;QACnF,CAAC;IACF,CAAC;IAED,yBAAyB;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAA;IAE9B,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAE,EAAE,MAAM,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAA;IACjE,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC"}
|
|
@@ -44,11 +44,6 @@ export declare const FCC_BDC_ADAPTER_ID = "fcc-bdc";
|
|
|
44
44
|
* than having to look them up.
|
|
45
45
|
*/
|
|
46
46
|
export declare const FCC_BDC_DEFAULT_LICENSE = "Public Domain";
|
|
47
|
-
interface SplitAddress {
|
|
48
|
-
house_number?: string;
|
|
49
|
-
street: string;
|
|
50
|
-
}
|
|
51
|
-
export declare function splitAddressPrimary(address: string): SplitAddress | null;
|
|
52
47
|
/**
|
|
53
48
|
* Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
|
|
54
49
|
*
|
|
@@ -70,5 +65,4 @@ export declare function createFccBdcAdapter(): CorpusAdapter;
|
|
|
70
65
|
* The configured adapter instance registered with the corpus builder.
|
|
71
66
|
*/
|
|
72
67
|
export declare const fccBdcAdapter: CorpusAdapter;
|
|
73
|
-
export {};
|
|
74
68
|
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAOH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEjF;;;GAGG;AACH,eAAO,MAAM,kBAAkB,YAAY,CAAA;AAC3C;;;GAGG;AACH,eAAO,MAAM,uBAAuB,kBAAkB,CAAA;AActD;;;;;;;;;;;GAWG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,CAWxE;AAED;;GAEG;AACH,wBAAgB,mBAAmB,IAAI,aAAa,CAyEnD;AAED;;GAEG;AACH,eAAO,MAAM,aAAa,eAAwB,CAAA"}
|
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
* re-stamp accordingly.
|
|
35
35
|
*/
|
|
36
36
|
import { DatabaseSync } from "node:sqlite";
|
|
37
|
+
import { splitStreetLine } from "../../adapter.js";
|
|
37
38
|
import { lookupStateAbbreviation } from "../../codex/us-fips-state.js";
|
|
38
39
|
import { formatAddress, reconcileComponents } from "../../format.js";
|
|
39
40
|
/**
|
|
@@ -46,29 +47,6 @@ export const FCC_BDC_ADAPTER_ID = "fcc-bdc";
|
|
|
46
47
|
* than having to look them up.
|
|
47
48
|
*/
|
|
48
49
|
export const FCC_BDC_DEFAULT_LICENSE = "Public Domain";
|
|
49
|
-
/**
|
|
50
|
-
* Split `address_primary` into a `(house_number, street)` pair.
|
|
51
|
-
*
|
|
52
|
-
* BDC's `address_primary` follows USPS Publication 28 conventions but with hand-entry drift. The canonical
|
|
53
|
-
* leading-digit prefix is the house number (`"123 Main St"`, `"6450 W Indian School Rd"`, even hyphenated forms `"40-12
|
|
54
|
-
* Bell Blvd"`). Anything that doesn't match the prefix shape (`"PO Box 1234"`, `"RR 2 Box 67"`, `"HC 1"`) is left as a
|
|
55
|
-
* single `street` value — the model sees the original surface form, and downstream classifiers/po-box handling can pick
|
|
56
|
-
* it up.
|
|
57
|
-
*
|
|
58
|
-
* The regex tolerates one trailing letter (`"123A Main St"`) and an optional hyphenated half (`"40-12"`) which is
|
|
59
|
-
* common in NYC + suburban garden-apartment numbering.
|
|
60
|
-
*/
|
|
61
|
-
const HOUSE_NUMBER_PREFIX = /^(\d+(?:-\d+)?[A-Za-z]?)\s+(.+)$/;
|
|
62
|
-
export function splitAddressPrimary(address) {
|
|
63
|
-
const trimmed = address.trim();
|
|
64
|
-
if (!trimmed)
|
|
65
|
-
return null;
|
|
66
|
-
const match = HOUSE_NUMBER_PREFIX.exec(trimmed);
|
|
67
|
-
if (match) {
|
|
68
|
-
return { house_number: match[1], street: match[2].trim() };
|
|
69
|
-
}
|
|
70
|
-
return { street: trimmed };
|
|
71
|
-
}
|
|
72
50
|
/**
|
|
73
51
|
* Combine `zip` + optional `zip_suffix` into the canonical USPS postcode surface form.
|
|
74
52
|
*
|
|
@@ -115,7 +93,7 @@ export function createFccBdcAdapter() {
|
|
|
115
93
|
return;
|
|
116
94
|
if (opts.limit !== undefined && emitted >= opts.limit)
|
|
117
95
|
return;
|
|
118
|
-
const split =
|
|
96
|
+
const split = splitStreetLine(row.address_primary ?? "");
|
|
119
97
|
if (!split)
|
|
120
98
|
continue;
|
|
121
99
|
const state = lookupStateAbbreviation(row.state);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAE1C,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAGpE;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAC3C;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAA;AActD;;;;;;;;;;;GAWG;AACH,MAAM,
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/fcc-bdc/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAA;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAA;AACtE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,iBAAiB,CAAA;AAGpE;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAG,SAAS,CAAA;AAC3C;;;GAGG;AACH,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAA;AActD;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,aAAa,CAAC,GAAW,EAAE,MAAqB;IAC/D,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAEpB,IAAI,CAAC,CAAC;QAAE,OAAO,EAAE,CAAA;IACjB,MAAM,CAAC,GAAG,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;IAE9B,IAAI,CAAC,CAAC;QAAE,OAAO,CAAC,CAAA;IAEhB,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAA;IAE7B,OAAO,GAAG,CAAC,IAAI,CAAC,EAAE,CAAA;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,mBAAmB;IAClC,OAAO;QACN,EAAE,EAAE,kBAAkB;QACtB,cAAc,EAAE,uBAAuB;QACvC,WAAW,EACV,wIAAwI;QAEzI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBAC3C,MAAM,IAAI,KAAK,CAAC,mDAAmD,IAAI,CAAC,OAAO,EAAE,CAAC,CAAA;YACnF,CAAC;YAED,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAA;YAC/D,IAAI,OAAO,GAAG,CAAC,CAAA;YAEf,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,CACtB;;2BAEsB,CACtB,CAAA;gBAED,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,EAAsC,EAAE,CAAC;oBACtE,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;wBAAE,OAAM;oBAEhC,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,IAAI,CAAC,KAAK;wBAAE,OAAM;oBAE7D,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,eAAe,IAAI,EAAE,CAAC,CAAA;oBAExD,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBACpB,MAAM,KAAK,GAAG,uBAAuB,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;oBAEhD,IAAI,CAAC,KAAK;wBAAE,SAAQ;oBACpB,MAAM,QAAQ,GAAG,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,CAAA;oBAEjC,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBACvB,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,EAAE,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,CAAA;oBAErE,IAAI,CAAC,QAAQ;wBAAE,SAAQ;oBAEvB,MAAM,UAAU,GAA+B;wBAC9C,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBACnE,MAAM,EAAE,KAAK,CAAC,MAAM;wBACpB,QAAQ;wBACR,MAAM,EAAE,KAAK,CAAC,YAAY;wBAC1B,QAAQ;qBACR,CAAA;oBAED,MAAM,GAAG,GAAG,aAAa,CAAC,UAAU,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBAEhE,IAAI,CAAC,GAAG;wBAAE,SAAQ;oBAClB,MAAM,OAAO,GAAG,mBAAmB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;oBAEpD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM;wBAAE,SAAQ;oBAE1C,MAAM;wBACL,GAAG;wBACH,UAAU,EAAE,OAAO;wBACnB,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,OAAO;wBACf,MAAM,EAAE,kBAAkB;wBAC1B,SAAS,EAAE,GAAG,kBAAkB,IAAI,GAAG,CAAC,WAAW,EAAE;wBACrD,cAAc,EAAE,EAAE;wBAClB,OAAO,EAAE,uBAAuB;qBAChC,CAAA;oBAED,OAAO,EAAE,CAAA;gBACV,CAAC;YACF,CAAC;oBAAS,CAAC;gBACV,EAAE,CAAC,KAAK,EAAE,CAAA;YACX,CAAC;QACF,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,mBAAmB,EAAE,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/geonames/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEjF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,aAAa,CAAA;AAC7C;;;GAGG;AACH,eAAO,MAAM,wBAAwB,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/geonames/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AASH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAEjF;;;GAGG;AACH,eAAO,MAAM,mBAAmB,aAAa,CAAA;AAC7C;;;GAGG;AACH,eAAO,MAAM,wBAAwB,cAAc,CAAA;AA8DnD,wBAAgB,qBAAqB,IAAI,aAAa,CAoFrD;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,eAA0B,CAAA"}
|