@mailwoman/corpus 4.16.2 → 5.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +23 -26
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +25 -28
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +10 -10
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts +2 -2
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +2 -2
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +2 -2
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +9 -9
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +5 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +3 -3
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -10
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +4 -5
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +9 -10
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +13 -14
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +8 -10
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +8 -8
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +6 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +6 -6
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +7 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +9 -9
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +6 -6
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts +2 -2
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +2 -2
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +15 -17
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +6 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -6
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +7 -7
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +12 -12
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +3 -3
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +6 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +17 -19
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +25 -28
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +8 -8
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +11 -11
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +14 -16
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +15 -16
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +13 -15
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +6 -6
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -8
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +6 -8
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +1 -1
- package/out/src/golden.js.map +1 -1
- package/out/src/index.d.ts +1 -0
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/license.d.ts +5 -6
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +5 -6
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +2 -2
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +16 -18
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +22 -23
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +10 -10
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +8 -8
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +16 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -0
- package/out/src/shard-recipes/anchor-absorption.js +73 -0
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts +16 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -0
- package/out/src/shard-recipes/boundary-stress.js +77 -0
- package/out/src/shard-recipes/boundary-stress.js.map +1 -0
- package/out/src/shard-recipes/country-balanced.d.ts +28 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -0
- package/out/src/shard-recipes/country-balanced.js +370 -0
- package/out/src/shard-recipes/country-balanced.js.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts +34 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.js +175 -0
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts +24 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.js +76 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -0
- package/out/src/shard-recipes/fr-order.d.ts +34 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-order.js +238 -0
- package/out/src/shard-recipes/fr-order.js.map +1 -0
- package/out/src/shard-recipes/german.d.ts +19 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -0
- package/out/src/shard-recipes/german.js +170 -0
- package/out/src/shard-recipes/german.js.map +1 -0
- package/out/src/shard-recipes/house-venue.d.ts +14 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/house-venue.js +66 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts +18 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -0
- package/out/src/shard-recipes/index.js +58 -0
- package/out/src/shard-recipes/index.js.map +1 -0
- package/out/src/shard-recipes/intersection.d.ts +38 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -0
- package/out/src/shard-recipes/intersection.js +473 -0
- package/out/src/shard-recipes/intersection.js.map +1 -0
- package/out/src/shard-recipes/locale.d.ts +19 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -0
- package/out/src/shard-recipes/locale.js +257 -0
- package/out/src/shard-recipes/locale.js.map +1 -0
- package/out/src/shard-recipes/no-street.d.ts +14 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -0
- package/out/src/shard-recipes/no-street.js +66 -0
- package/out/src/shard-recipes/no-street.js.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts +34 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.js +686 -0
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -0
- package/out/src/shard-recipes/po-box.d.ts +13 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box.js +93 -0
- package/out/src/shard-recipes/po-box.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +104 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -0
- package/out/src/shard-recipes/scaffold.js +76 -0
- package/out/src/shard-recipes/scaffold.js.map +1 -0
- package/out/src/shard-recipes/street-affix.d.ts +31 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -0
- package/out/src/shard-recipes/street-affix.js +408 -0
- package/out/src/shard-recipes/street-affix.js.map +1 -0
- package/out/src/shard-recipes/street-bare.d.ts +21 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -0
- package/out/src/shard-recipes/street-bare.js +82 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -0
- package/out/src/shard-recipes/street.d.ts +11 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -0
- package/out/src/shard-recipes/street.js +63 -0
- package/out/src/shard-recipes/street.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +25 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -0
- package/out/src/shard-recipes/unit.js +255 -0
- package/out/src/shard-recipes/unit.js.map +1 -0
- package/out/src/split.d.ts +23 -29
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +22 -29
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +2 -3
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +2 -2
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -4
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +3 -4
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -19
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +14 -17
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts +2 -3
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +2 -3
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +2 -2
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +4 -4
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +5 -5
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +27 -23
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -17
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +27 -27
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +8 -9
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +3 -3
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +57 -66
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +56 -65
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +5 -6
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js +5 -6
- package/out/src/tokenize.js.map +1 -1
- package/out/src/types.d.ts +49 -51
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +32 -34
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +23 -25
- package/out/src/wof-json.js.map +1 -1
- package/package.json +14 -14
package/out/src/wof-json.d.ts
CHANGED
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
*
|
|
16
16
|
* This module provides:
|
|
17
17
|
*
|
|
18
|
-
* - `
|
|
19
|
-
* - `walkFeatures`: streaming directory walk → parsed `
|
|
18
|
+
* - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
|
|
19
|
+
* - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
|
|
20
20
|
* deprecated records).
|
|
21
21
|
* - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
|
|
22
22
|
* - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
|
|
@@ -29,16 +29,16 @@
|
|
|
29
29
|
* silently emitted zero rows from the real corpus.
|
|
30
30
|
*/
|
|
31
31
|
/** A WOF GeoJSON feature, as published by the per-record bundles. */
|
|
32
|
-
export interface
|
|
32
|
+
export interface WOFFeature {
|
|
33
33
|
type?: string;
|
|
34
34
|
id?: number | string;
|
|
35
35
|
properties?: Record<string, unknown> | null;
|
|
36
36
|
}
|
|
37
37
|
/**
|
|
38
|
-
* Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped —
|
|
39
|
-
*
|
|
38
|
+
* Lightweight in-memory shape both adapters keep per record. Geometry is intentionally dropped — it's 95% of the file
|
|
39
|
+
* weight and the adapters never consult it.
|
|
40
40
|
*/
|
|
41
|
-
export interface
|
|
41
|
+
export interface WOFRecord {
|
|
42
42
|
id: number;
|
|
43
43
|
parent_id: number | null;
|
|
44
44
|
/** Canonical `wof:name` of the record. */
|
|
@@ -49,57 +49,55 @@ export interface WofRecord {
|
|
|
49
49
|
/**
|
|
50
50
|
* Localized name variants from `name:*` properties.
|
|
51
51
|
*
|
|
52
|
-
* Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the
|
|
53
|
-
*
|
|
54
|
-
*
|
|
52
|
+
* Keys are the raw `name:eng_x_preferred` form; values are the first non-empty string from the underlying array (WOF
|
|
53
|
+
* stores variants as arrays even when only one form is present). The canonical `wof:name` is NOT included here —
|
|
54
|
+
* adapters add a synthetic `"default"` slot for it.
|
|
55
55
|
*/
|
|
56
56
|
nameVariants: Map<string, string>;
|
|
57
57
|
}
|
|
58
58
|
/**
|
|
59
59
|
* `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
|
|
60
60
|
*
|
|
61
|
-
* Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
* fix.
|
|
61
|
+
* Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
|
|
62
|
+
* alongside `1`. The previous SpatiaLite-backed adapters filtered on `= 1` only and silently emitted zero rows from the
|
|
63
|
+
* corpus — this loosened predicate is the critical fix.
|
|
65
64
|
*/
|
|
66
65
|
export declare function isCurrentFeature(props: Record<string, unknown>): boolean;
|
|
67
66
|
/**
|
|
68
|
-
* Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* by iterating the underlying array if they need it.
|
|
67
|
+
* Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
|
|
68
|
+
* Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
|
|
69
|
+
* not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
|
|
72
70
|
*/
|
|
73
71
|
export declare function extractNameVariants(props: Record<string, unknown>): Map<string, string>;
|
|
74
72
|
/**
|
|
75
73
|
* Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
|
|
76
74
|
*
|
|
77
|
-
* `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both
|
|
78
|
-
*
|
|
75
|
+
* `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
|
|
76
|
+
* existing source_id separator vocabulary and downstream consumers split on `-`.
|
|
79
77
|
*/
|
|
80
78
|
export declare function normalizeNameKey(rawKey: string): string;
|
|
81
79
|
/**
|
|
82
|
-
* Stream every canonical GeoJSON file under `repoDir` and yield parsed `
|
|
80
|
+
* Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
|
|
83
81
|
*
|
|
84
|
-
* `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
82
|
+
* `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
|
|
83
|
+
* (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
|
|
84
|
+
* `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
|
|
85
|
+
* records.
|
|
88
86
|
*
|
|
89
|
-
* Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file
|
|
90
|
-
*
|
|
87
|
+
* Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
|
|
88
|
+
* walk. Adapters can add stricter validation downstream if they need it.
|
|
91
89
|
*/
|
|
92
90
|
export declare function walkFeatures(repoDir: string, opts?: {
|
|
93
91
|
signal?: AbortSignal;
|
|
94
|
-
}): AsyncIterable<
|
|
92
|
+
}): AsyncIterable<WOFRecord>;
|
|
95
93
|
/**
|
|
96
|
-
* Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id`
|
|
97
|
-
*
|
|
98
|
-
*
|
|
94
|
+
* Build an in-memory ancestry index: `Map<wof_id, [parent, grandparent, ...]>` walking `parent_id` upward and stopping
|
|
95
|
+
* at the first missing link. A cycle guard halts at any re-visit (defensive — WOF data is acyclic by construction but
|
|
96
|
+
* corrupt fixtures shouldn't infinite-loop the adapter).
|
|
99
97
|
*
|
|
100
|
-
* Records whose ancestors aren't in `
|
|
101
|
-
*
|
|
98
|
+
* Records whose ancestors aren't in `byID` (e.g. an FR locality whose region wasn't included in the cloned repo set)
|
|
99
|
+
* get a shorter chain; the variant emission gracefully degrades.
|
|
102
100
|
*/
|
|
103
|
-
export type AncestryIndex = Map<number,
|
|
104
|
-
export declare function buildAncestryIndex(
|
|
101
|
+
export type AncestryIndex = Map<number, WOFRecord[]>;
|
|
102
|
+
export declare function buildAncestryIndex(byID: Map<number, WOFRecord>): AncestryIndex;
|
|
105
103
|
//# sourceMappingURL=wof-json.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;
|
|
1
|
+
{"version":3,"file":"wof-json.d.ts","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAMH,qEAAqE;AACrE,MAAM,WAAW,UAAU;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,IAAI,CAAA;CAC3C;AAED;;;GAGG;AACH,MAAM,WAAW,SAAS;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,0CAA0C;IAC1C,IAAI,EAAE,MAAM,CAAA;IACZ,SAAS,EAAE,MAAM,CAAA;IACjB,6CAA6C;IAC7C,OAAO,EAAE,MAAM,CAAA;IACf;;;;;;OAMG;IACH,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACjC;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAKxE;AAED;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAevF;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEvD;AA4CD;;;;;;;;;;GAUG;AACH,wBAAuB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAO,GAAG,aAAa,CAAC,SAAS,CAAC,CAkClH;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,EAAE,CAAC,CAAA;AAEpD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,SAAS,CAAC,GAAG,aAAa,CAsB9E"}
|
package/out/src/wof-json.js
CHANGED
|
@@ -15,8 +15,8 @@
|
|
|
15
15
|
*
|
|
16
16
|
* This module provides:
|
|
17
17
|
*
|
|
18
|
-
* - `
|
|
19
|
-
* - `walkFeatures`: streaming directory walk → parsed `
|
|
18
|
+
* - `WOFRecord`: the lightweight per-feature shape both adapters carry in their ancestry index.
|
|
19
|
+
* - `walkFeatures`: streaming directory walk → parsed `WOFRecord`s (skips alt files, bad JSON,
|
|
20
20
|
* deprecated records).
|
|
21
21
|
* - `buildAncestryIndex`: in-memory ancestry chain construction (`Map<id, ancestors[]>`).
|
|
22
22
|
* - `extractNameVariants`: pulls `name:*` localized name lists off a feature's properties.
|
|
@@ -28,15 +28,14 @@
|
|
|
28
28
|
* treated as active"), which is why the previous SpatiaLite adapter's `is_current = 1` filter
|
|
29
29
|
* silently emitted zero rows from the real corpus.
|
|
30
30
|
*/
|
|
31
|
-
import FastGlob from "fast-glob";
|
|
32
31
|
import { readFile } from "node:fs/promises";
|
|
32
|
+
import FastGlob from "fast-glob";
|
|
33
33
|
/**
|
|
34
34
|
* `mz:is_current` ∈ {`1`, `-1`} → keep. `0` → drop.
|
|
35
35
|
*
|
|
36
|
-
* Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
* fix.
|
|
36
|
+
* Real WOF postalcode distros tag every row `-1` ("unknown but treated as active"); the Pelias importer accepts `-1`
|
|
37
|
+
* alongside `1`. The previous SpatiaLite-backed adapters filtered on `= 1` only and silently emitted zero rows from the
|
|
38
|
+
* corpus — this loosened predicate is the critical fix.
|
|
40
39
|
*/
|
|
41
40
|
export function isCurrentFeature(props) {
|
|
42
41
|
const raw = props["mz:is_current"];
|
|
@@ -44,10 +43,9 @@ export function isCurrentFeature(props) {
|
|
|
44
43
|
return n !== 0;
|
|
45
44
|
}
|
|
46
45
|
/**
|
|
47
|
-
* Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
* by iterating the underlying array if they need it.
|
|
46
|
+
* Pull `name:*` localized variants off a WOF feature's properties. WOF stores variants as arrays (`["Saint
|
|
47
|
+
* Petersburg"]`); we lift the first non-empty string. Multiple-value variants (rare; usually historical aliases) are
|
|
48
|
+
* not split into separate rows by this helper — adapters can opt in by iterating the underlying array if they need it.
|
|
51
49
|
*/
|
|
52
50
|
export function extractNameVariants(props) {
|
|
53
51
|
const out = new Map();
|
|
@@ -67,8 +65,8 @@ export function extractNameVariants(props) {
|
|
|
67
65
|
/**
|
|
68
66
|
* Turn a `name:*` property key into a hyphen-safe suffix fragment for `source_id`.
|
|
69
67
|
*
|
|
70
|
-
* `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both
|
|
71
|
-
*
|
|
68
|
+
* `"name:eng_x_colloquial"` → `"name-eng-x-colloquial"`. `:` and `_` both become `-` because both collide with the
|
|
69
|
+
* existing source_id separator vocabulary and downstream consumers split on `-`.
|
|
72
70
|
*/
|
|
73
71
|
export function normalizeNameKey(rawKey) {
|
|
74
72
|
return rawKey.replace(/[:_]/g, "-");
|
|
@@ -78,8 +76,8 @@ function recordFromFeature(feature) {
|
|
|
78
76
|
if (!feature || feature.type !== "Feature" || !feature.properties)
|
|
79
77
|
return null;
|
|
80
78
|
const props = feature.properties;
|
|
81
|
-
const
|
|
82
|
-
const id = typeof
|
|
79
|
+
const rawID = typeof feature.id === "number" ? feature.id : props["wof:id"];
|
|
80
|
+
const id = typeof rawID === "number" ? rawID : typeof rawID === "string" ? Number(rawID) : NaN;
|
|
83
81
|
if (!Number.isFinite(id))
|
|
84
82
|
return null;
|
|
85
83
|
const name = props["wof:name"];
|
|
@@ -109,15 +107,15 @@ function recordFromFeature(feature) {
|
|
|
109
107
|
};
|
|
110
108
|
}
|
|
111
109
|
/**
|
|
112
|
-
* Stream every canonical GeoJSON file under `repoDir` and yield parsed `
|
|
110
|
+
* Stream every canonical GeoJSON file under `repoDir` and yield parsed `WOFRecord`s.
|
|
113
111
|
*
|
|
114
|
-
* `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
112
|
+
* `repoDir` may point at a single cloned `whosonfirst-data-*` repo OR at a parent directory holding several such repos
|
|
113
|
+
* (the corpus pipeline clones all four into a shared `wof/repos/` root and runs the adapter against that root).
|
|
114
|
+
* `**\/*.geojson` walks the whole tree; `-alt-` siblings are skipped since they're alternate-geometry exports, not new
|
|
115
|
+
* records.
|
|
118
116
|
*
|
|
119
|
-
* Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file
|
|
120
|
-
*
|
|
117
|
+
* Errors per-file (unreadable, malformed JSON, missing properties) are swallowed so one bad file doesn't poison a 3 GB
|
|
118
|
+
* walk. Adapters can add stricter validation downstream if they need it.
|
|
121
119
|
*/
|
|
122
120
|
export async function* walkFeatures(repoDir, opts = {}) {
|
|
123
121
|
const stream = FastGlob.stream(["**/*.geojson"], {
|
|
@@ -151,14 +149,14 @@ export async function* walkFeatures(repoDir, opts = {}) {
|
|
|
151
149
|
yield rec;
|
|
152
150
|
}
|
|
153
151
|
}
|
|
154
|
-
export function buildAncestryIndex(
|
|
152
|
+
export function buildAncestryIndex(byID) {
|
|
155
153
|
const index = new Map();
|
|
156
|
-
for (const [id, rec] of
|
|
154
|
+
for (const [id, rec] of byID) {
|
|
157
155
|
const chain = [];
|
|
158
156
|
const guard = new Set([id]);
|
|
159
157
|
let cur = rec.parent_id;
|
|
160
158
|
while (cur !== null && cur > 0) {
|
|
161
|
-
const parent =
|
|
159
|
+
const parent = byID.get(cur);
|
|
162
160
|
if (!parent)
|
|
163
161
|
break;
|
|
164
162
|
if (guard.has(parent.id))
|
package/out/src/wof-json.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,QAAQ,MAAM,
|
|
1
|
+
{"version":3,"file":"wof-json.js","sourceRoot":"","sources":["../../src/wof-json.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AAE3C,OAAO,QAAQ,MAAM,WAAW,CAAA;AA+BhC;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAA8B;IAC9D,MAAM,GAAG,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAClC,MAAM,CAAC,GAAG,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAA8B;IACjE,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAA;IAErC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAClD,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,SAAQ;QACtC,MAAM,SAAS,GAAG,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;YACrC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;YAC9E,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACrD,CAAC,CAAC,KAAK;gBACP,CAAC,CAAC,SAAS,CAAA;QAEb,IAAI,SAAS;YAAE,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAC9C,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAC9C,OAAO,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;AACpC,CAAC;AAED,0FAA0F;AAC1F,SAAS,iBAAiB,CAAC,OAAmB;IAC7C,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,UAAU;QAAE,OAAO,IAAI,CAAA;IAC9E,MAAM,KAAK,GAAG,OAAO,CAAC,UAAU,CAAA;IAEhC,MAAM,KAAK,GAAG,OAAO,OAAO,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAA;IAC3E,MAAM,EAAE,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IAE9F,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,IAAI,GAAG,KAAK,CAAC,UAAU,CAAC,CAAA;IAE9B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE;QAAE,OAAO,IAAI,CAAA;IAEzD,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IAExC,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAA;IAE5D,MAAM,OAAO,GAAG,KAAK,CAAC,aAAa,CAAC,CAAA;IAEpC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAExD,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,KAAK,CAAC,eAAe,CAAC,CAAA;IACxC,MAAM,SAAS,GACd,OAAO,SAAS,KAAK,QAAQ;QAC5B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE;YAClD,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC;YACnB,CAAC,CAAC,IAAI,CAAA;IAET,OAAO;QACN,EAAE;QACF,SAAS,EAAE,MAAM,CAAC,QAAQ,CAAC,SAAmB,CAAC,CAAC,CAAC,CAAE,SAAoB,CAAC,CAAC,CAAC,IAAI;QAC9E,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;QACjB,SAAS;QACT,OAAO;QACP,YAAY,EAAE,mBAAmB,CAAC,KAAK,CAAC;KACxC,CAAA;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,YAAY,CAAC,OAAe,EAAE,OAAiC,EAAE;IACvF,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE;QAChD,GAAG,EAAE,OAAO;QACZ,QAAQ,EAAE,IAAI;QACd,SAAS,EAAE,IAAI;QACf,cAAc,EAAE,IAAI;KACpB,CAAC,CAAA;IAEF,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAChC,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC,CAAA;QAE9B,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC;YAAE,SAAQ;QAExC,IAAI,IAAY,CAAA;QAEhB,IAAI,CAAC;YACJ,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,IAAI,MAAkB,CAAA;QAEtB,IAAI,CAAC;YACJ,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAe,CAAA;QACxC,CAAC;QAAC,MAAM,CAAC;YACR,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAA;QAErC,IAAI,GAAG;YAAE,MAAM,GAAG,CAAA;IACnB,CAAC;AACF,CAAC;AAYD,MAAM,UAAU,kBAAkB,CAAC,IAA4B;IAC9D,MAAM,KAAK,GAAkB,IAAI,GAAG,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAgB,EAAE,CAAA;QAC7B,MAAM,KAAK,GAAG,IAAI,GAAG,CAAS,CAAC,EAAE,CAAC,CAAC,CAAA;QACnC,IAAI,GAAG,GAAkB,GAAG,CAAC,SAAS,CAAA;QAEtC,OAAO,GAAG,KAAK,IAAI,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,MAAM;gBAAE,MAAK;YAElB,IAAI,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAAE,MAAK;YAC/B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAClB,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACpB,GAAG,GAAG,MAAM,CAAC,SAAS,CAAA;QACvB,CAAC;QACD,KAAK,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IACrB,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mailwoman/corpus",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.1.0",
|
|
4
4
|
"description": "Mailwoman corpus pipeline: BIO-labeled dataset builder for the neural classifier.",
|
|
5
5
|
"license": "AGPL-3.0-only",
|
|
6
6
|
"repository": {
|
|
@@ -8,6 +8,12 @@
|
|
|
8
8
|
"url": "https://github.com/sister-software/mailwoman.git",
|
|
9
9
|
"directory": "corpus"
|
|
10
10
|
},
|
|
11
|
+
"files": [
|
|
12
|
+
"out/**/*.js",
|
|
13
|
+
"out/**/*.js.map",
|
|
14
|
+
"out/**/*.d.ts",
|
|
15
|
+
"out/**/*.d.ts.map"
|
|
16
|
+
],
|
|
11
17
|
"type": "module",
|
|
12
18
|
"exports": {
|
|
13
19
|
"./package.json": "./package.json",
|
|
@@ -15,27 +21,21 @@
|
|
|
15
21
|
"./types": "./out/src/types.js",
|
|
16
22
|
"./format": "./out/src/format.js"
|
|
17
23
|
},
|
|
24
|
+
"publishConfig": {
|
|
25
|
+
"access": "public"
|
|
26
|
+
},
|
|
18
27
|
"dependencies": {
|
|
19
28
|
"@dsnp/parquetjs": "1.8.7",
|
|
20
|
-
"@mailwoman/codex": "
|
|
21
|
-
"@mailwoman/core": "
|
|
22
|
-
"@mailwoman/formatter": "
|
|
29
|
+
"@mailwoman/codex": "5.1.0",
|
|
30
|
+
"@mailwoman/core": "5.1.0",
|
|
31
|
+
"@mailwoman/formatter": "5.1.0",
|
|
23
32
|
"csv-parse": "^5.6.0",
|
|
24
33
|
"fast-glob": "^3.3.3",
|
|
25
34
|
"fastest-levenshtein": "^1.0.16",
|
|
26
35
|
"lru-cache": "^10.4.3",
|
|
27
|
-
"spliterator": "^
|
|
36
|
+
"spliterator": "^3.1.0"
|
|
28
37
|
},
|
|
29
38
|
"devDependencies": {
|
|
30
39
|
"@types/node": ">=24.18.0"
|
|
31
|
-
},
|
|
32
|
-
"files": [
|
|
33
|
-
"out/**/*.js",
|
|
34
|
-
"out/**/*.js.map",
|
|
35
|
-
"out/**/*.d.ts",
|
|
36
|
-
"out/**/*.d.ts.map"
|
|
37
|
-
],
|
|
38
|
-
"publishConfig": {
|
|
39
|
-
"access": "public"
|
|
40
40
|
}
|
|
41
41
|
}
|