@mailwoman/corpus 9.3.0 → 10.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +16 -0
- package/lib/adapters/geonames/adapter.ts +3 -3
- package/lib/adapters/geonames/postal/adapter.ts +122 -0
- package/lib/adapters/index.ts +39 -32
- package/lib/adapters/openaddresses/adapter.ts +4 -7
- package/lib/adapters/osm/adapter.ts +278 -0
- package/lib/adapters/overture/adapter.ts +22 -7
- package/lib/adapters/utils/index.ts +1 -1
- package/lib/adapters/wof/admin/json/adapter.ts +239 -0
- package/lib/adapters/wof/json-rows.ts +181 -0
- package/lib/adapters/wof/postalcode-json/adapter.ts +196 -0
- package/lib/au/adapters/gnaf/adapter.ts +152 -0
- package/lib/au/adapters/gnaf/assemble.ts +225 -0
- package/lib/cz/recipes/pcfirst-preposition.ts +112 -0
- package/lib/de/recipes/locale.ts +360 -0
- package/lib/fr/adapters/ban/adapter.ts +159 -0
- package/lib/fr/recipes/admin-split.ts +243 -0
- package/lib/fr/recipes/bare-street.ts +171 -0
- package/lib/fr/recipes/fragment.ts +334 -0
- package/lib/fr/recipes/lieudit.ts +274 -0
- package/lib/fr/recipes/order.ts +246 -0
- package/lib/fr/tools/fetch/ban.ts +289 -0
- package/lib/index.ts +1 -1
- package/lib/international/recipes/locale.ts +598 -0
- package/lib/jp/tools/fetch/houjin.ts +141 -0
- package/lib/kr/adapters/juso/label-rows.ts +259 -0
- package/lib/kr/adapters/localdata/align.ts +299 -0
- package/lib/kr/adapters/localdata/key-index.ts +185 -0
- package/lib/kr/adapters/localdata/permit-rows.ts +186 -0
- package/lib/kr/tools/fetch/juso.ts +172 -0
- package/lib/kr/tools/fetch/localdata.ts +206 -0
- package/lib/nl/recipes/postcode.ts +117 -0
- package/lib/no/recipes/fragment.ts +213 -0
- package/lib/no/recipes/street/index.ts +91 -0
- package/lib/no/recipes/street/led.ts +135 -0
- package/lib/recipes/anchor-absorption.ts +2 -2
- package/lib/recipes/bare/country.ts +95 -0
- package/lib/recipes/bare/postcode/eval.ts +99 -0
- package/lib/recipes/bare/postcode/index.ts +327 -0
- package/lib/recipes/boundary-stress.ts +3 -1
- package/lib/recipes/country-balanced.ts +3 -2
- package/lib/recipes/house-venue.ts +2 -1
- package/lib/recipes/index.ts +28 -21
- package/lib/recipes/intersection.ts +3 -2
- package/lib/recipes/po/box/cedex/au-nz.ts +82 -0
- package/lib/recipes/po/box/cedex/ca.ts +58 -0
- package/lib/recipes/po/box/cedex/fr.ts +76 -0
- package/lib/recipes/po/box/cedex/phrases.ts +108 -0
- package/lib/recipes/po/box/cedex/recipe.ts +203 -0
- package/lib/recipes/po/box/cedex/sources.ts +189 -0
- package/lib/recipes/po/box/cedex/types.ts +35 -0
- package/lib/recipes/po/box/cedex/us.ts +69 -0
- package/lib/recipes/po/box/cedex/vocabulary.ts +106 -0
- package/lib/recipes/po/box/cedex.ts +9 -0
- package/lib/recipes/po/box/index.ts +128 -0
- package/lib/recipes/scaffold.ts +18 -42
- package/lib/recipes/street/affix.ts +807 -0
- package/lib/recipes/street/bare.ts +102 -0
- package/lib/recipes/street.ts +3 -1
- package/lib/recipes/sub/venue/context.ts +149 -0
- package/lib/recipes/sub/venue/index.ts +997 -0
- package/lib/recipes/sub/venue/sources.ts +739 -0
- package/lib/recipes/trailing-region.ts +9 -9
- package/lib/recipes/unit.ts +3 -3
- package/lib/sg/recipes/register.ts +270 -0
- package/lib/sg/tools/fetch/acra.ts +179 -0
- package/lib/si/recipes/bare-village.ts +99 -0
- package/lib/south-asia/recipes/register.ts +214 -0
- package/lib/synthesizers/anchor-absorption.ts +1 -1
- package/lib/synthesizers/locale.ts +261 -0
- package/lib/synthesizers/street.ts +1 -1
- package/lib/synthesizers/utils.ts +2 -2
- package/lib/tools/audit.ts +3 -2
- package/lib/tools/corpus-stats.ts +3 -3
- package/lib/tools/fetch/download/index.ts +10 -0
- package/lib/tools/fetch/download/manifest.ts +102 -0
- package/lib/tools/fetch/download/network.ts +226 -0
- package/lib/tools/fetch/download/retry.ts +173 -0
- package/lib/tools/fetch/geonames/dump.ts +257 -0
- package/lib/tools/fetch/geonames/postal.ts +180 -0
- package/lib/tools/fetch/index.ts +47 -20
- package/lib/tools/fetch/openaddresses.ts +2 -2
- package/lib/tools/fetch/ourairports.ts +2 -2
- package/lib/tools/fetch/ppd.ts +1 -3
- package/lib/tools/fetch/wikidata-subvenue.ts +2 -2
- package/lib/tools/golden/promote.ts +340 -0
- package/lib/tools/golden/relabel-street/directory.ts +340 -0
- package/lib/tools/golden/relabel-street/index.ts +9 -0
- package/lib/tools/golden/relabel-street/row.ts +361 -0
- package/lib/tools/index.ts +10 -10
- package/lib/tools/ingest-csv.ts +5 -7
- package/lib/tools/lint/slice/vocab.ts +376 -0
- package/lib/tools/overlay-manifest.ts +92 -25
- package/lib/tools/overture-subvenue.ts +1 -1
- package/lib/tools/postcode-triples.ts +138 -14
- package/lib/tools/sub/venue/harvest.ts +234 -0
- package/lib/tools/sub/venue/head-nouns.ts +245 -0
- package/lib/tools/sub/venue/lexicon.ts +429 -0
- package/lib/tools/sub/venue/surfaces.ts +95 -0
- package/lib/tools/sub/venue/table.ts +281 -0
- package/lib/tools/sub/venue/wikidata.ts +85 -0
- package/lib/tw/tools/fetch/gcis.ts +180 -0
- package/lib/types.ts +1 -1
- package/lib/us/adapters/fcc-bdc/adapter.ts +173 -0
- package/lib/us/adapters/state/hi-schools/adapter.ts +160 -0
- package/lib/us/adapters/state/ia-contractors/adapter.ts +136 -0
- package/lib/us/adapters/state/ny-notaries/adapter.ts +133 -0
- package/lib/us/adapters/state/tx-notaries/adapter.ts +149 -0
- package/lib/us/adapters/tiger/adapter.ts +250 -0
- package/lib/us/adapters/usgov/hrsa-fqhc/adapter.ts +156 -0
- package/lib/us/adapters/usgov/imls-pls/adapter.ts +134 -0
- package/lib/us/adapters/usgov/irs-bmf/adapter.ts +145 -0
- package/lib/us/adapters/usgov/nad/adapter.ts +324 -0
- package/lib/us/adapters/usgov/nppes/adapter.ts +143 -0
- package/lib/us/adapters/usgov/samhsa-treatment-locator/adapter.ts +182 -0
- package/lib/us/tools/fetch/hrsa.ts +59 -0
- package/lib/us/tools/fetch/imls-pls.ts +149 -0
- package/lib/us/tools/fetch/nad.ts +341 -0
- package/lib/us/tools/fetch/nppes.ts +165 -0
- package/lib/us/tools/fetch/state/hi-schools.ts +161 -0
- package/lib/us/tools/fetch/state/sources.ts +146 -0
- package/lib/us/tools/fetch/tiger-full.ts +308 -0
- package/lib/utils/align.ts +1 -1
- package/lib/utils/golden.ts +12 -8
- package/lib/utils/wof-json.ts +7 -13
- package/out/adapters/geonames/adapter.d.ts +1 -1
- package/out/adapters/geonames/adapter.js +3 -3
- package/out/adapters/geonames/adapter.js.map +1 -1
- package/out/adapters/geonames/postal/adapter.d.ts.map +1 -0
- package/out/adapters/geonames/postal/adapter.js +107 -0
- package/out/adapters/geonames/postal/adapter.js.map +1 -0
- package/out/adapters/index.d.ts +16 -16
- package/out/adapters/index.d.ts.map +1 -1
- package/out/adapters/index.js +34 -32
- package/out/adapters/index.js.map +1 -1
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/adapters/openaddresses/adapter.js +4 -6
- package/out/adapters/openaddresses/adapter.js.map +1 -1
- package/out/adapters/osm/adapter.d.ts +95 -0
- package/out/adapters/osm/adapter.d.ts.map +1 -0
- package/out/adapters/osm/adapter.js +211 -0
- package/out/adapters/osm/adapter.js.map +1 -0
- package/out/adapters/overture/adapter.d.ts +10 -0
- package/out/adapters/overture/adapter.d.ts.map +1 -1
- package/out/adapters/overture/adapter.js +20 -7
- package/out/adapters/overture/adapter.js.map +1 -1
- package/out/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/adapters/utils/index.d.ts +1 -1
- package/out/adapters/utils/index.d.ts.map +1 -1
- package/out/adapters/utils/index.js.map +1 -1
- package/out/adapters/wof/admin/json/adapter.d.ts +79 -0
- package/out/adapters/wof/admin/json/adapter.d.ts.map +1 -0
- package/out/adapters/wof/admin/json/adapter.js +204 -0
- package/out/adapters/wof/admin/json/adapter.js.map +1 -0
- package/out/adapters/wof/json-rows.d.ts +73 -0
- package/out/adapters/wof/json-rows.d.ts.map +1 -0
- package/out/adapters/wof/json-rows.js +115 -0
- package/out/adapters/wof/json-rows.js.map +1 -0
- package/out/adapters/wof/postalcode-json/adapter.d.ts +65 -0
- package/out/adapters/wof/postalcode-json/adapter.d.ts.map +1 -0
- package/out/adapters/wof/postalcode-json/adapter.js +167 -0
- package/out/adapters/wof/postalcode-json/adapter.js.map +1 -0
- package/out/au/adapters/gnaf/adapter.d.ts.map +1 -0
- package/out/au/adapters/gnaf/adapter.js +127 -0
- package/out/au/adapters/gnaf/adapter.js.map +1 -0
- package/out/au/adapters/gnaf/assemble.d.ts +65 -0
- package/out/au/adapters/gnaf/assemble.d.ts.map +1 -0
- package/out/au/adapters/gnaf/assemble.js +154 -0
- package/out/au/adapters/gnaf/assemble.js.map +1 -0
- package/out/cn/recipes/organizational-units.d.ts +36 -0
- package/out/cn/recipes/organizational-units.d.ts.map +1 -0
- package/out/cn/recipes/organizational-units.js +159 -0
- package/out/cn/recipes/organizational-units.js.map +1 -0
- package/out/cz/recipes/pcfirst-preposition.d.ts +22 -0
- package/out/cz/recipes/pcfirst-preposition.d.ts.map +1 -0
- package/out/cz/recipes/pcfirst-preposition.js +101 -0
- package/out/cz/recipes/pcfirst-preposition.js.map +1 -0
- package/out/de/recipes/locale.d.ts +44 -0
- package/out/de/recipes/locale.d.ts.map +1 -0
- package/out/de/recipes/locale.js +289 -0
- package/out/de/recipes/locale.js.map +1 -0
- package/out/env.d.ts +1 -1
- package/out/fr/adapters/ban/adapter.d.ts.map +1 -0
- package/out/fr/adapters/ban/adapter.js +123 -0
- package/out/fr/adapters/ban/adapter.js.map +1 -0
- package/out/fr/adapters/ban/street-decompose.d.ts.map +1 -0
- package/out/fr/adapters/ban/street-decompose.js.map +1 -0
- package/out/fr/recipes/admin-split.d.ts +38 -0
- package/out/fr/recipes/admin-split.d.ts.map +1 -0
- package/out/fr/recipes/admin-split.js +192 -0
- package/out/fr/recipes/admin-split.js.map +1 -0
- package/out/fr/recipes/bare-street.d.ts +28 -0
- package/out/fr/recipes/bare-street.d.ts.map +1 -0
- package/out/fr/recipes/bare-street.js +144 -0
- package/out/fr/recipes/bare-street.js.map +1 -0
- package/out/fr/recipes/fragment.d.ts +77 -0
- package/out/fr/recipes/fragment.d.ts.map +1 -0
- package/out/fr/recipes/fragment.js +277 -0
- package/out/fr/recipes/fragment.js.map +1 -0
- package/out/fr/recipes/lieudit.d.ts +32 -0
- package/out/fr/recipes/lieudit.d.ts.map +1 -0
- package/out/fr/recipes/lieudit.js +211 -0
- package/out/fr/recipes/lieudit.js.map +1 -0
- package/out/fr/recipes/order.d.ts +38 -0
- package/out/fr/recipes/order.d.ts.map +1 -0
- package/out/fr/recipes/order.js +198 -0
- package/out/fr/recipes/order.js.map +1 -0
- package/out/fr/tools/fetch/ban.d.ts +31 -0
- package/out/fr/tools/fetch/ban.d.ts.map +1 -0
- package/out/fr/tools/fetch/ban.js +239 -0
- package/out/fr/tools/fetch/ban.js.map +1 -0
- package/out/index.d.ts +1 -1
- package/out/index.d.ts.map +1 -1
- package/out/index.js +1 -1
- package/out/index.js.map +1 -1
- package/out/international/recipes/locale.d.ts +136 -0
- package/out/international/recipes/locale.d.ts.map +1 -0
- package/out/international/recipes/locale.js +458 -0
- package/out/international/recipes/locale.js.map +1 -0
- package/out/jp/adapters/wof-admin/adapter.d.ts.map +1 -0
- package/out/jp/adapters/wof-admin/adapter.js.map +1 -0
- package/out/jp/tools/fetch/houjin.d.ts +24 -0
- package/out/jp/tools/fetch/houjin.d.ts.map +1 -0
- package/out/jp/tools/fetch/houjin.js +102 -0
- package/out/jp/tools/fetch/houjin.js.map +1 -0
- package/out/kr/adapters/juso/label-rows.d.ts +76 -0
- package/out/kr/adapters/juso/label-rows.d.ts.map +1 -0
- package/out/kr/adapters/juso/label-rows.js +194 -0
- package/out/kr/adapters/juso/label-rows.js.map +1 -0
- package/out/kr/adapters/localdata/align.d.ts +43 -0
- package/out/kr/adapters/localdata/align.d.ts.map +1 -0
- package/out/kr/adapters/localdata/align.js +223 -0
- package/out/kr/adapters/localdata/align.js.map +1 -0
- package/out/kr/adapters/localdata/key-index.d.ts +71 -0
- package/out/kr/adapters/localdata/key-index.d.ts.map +1 -0
- package/out/kr/adapters/localdata/key-index.js +133 -0
- package/out/kr/adapters/localdata/key-index.js.map +1 -0
- package/out/kr/adapters/localdata/permit-rows.d.ts +66 -0
- package/out/kr/adapters/localdata/permit-rows.d.ts.map +1 -0
- package/out/kr/adapters/localdata/permit-rows.js +131 -0
- package/out/kr/adapters/localdata/permit-rows.js.map +1 -0
- package/out/kr/tools/fetch/juso.d.ts +30 -0
- package/out/kr/tools/fetch/juso.d.ts.map +1 -0
- package/out/kr/tools/fetch/juso.js +120 -0
- package/out/kr/tools/fetch/juso.js.map +1 -0
- package/out/kr/tools/fetch/localdata.d.ts +30 -0
- package/out/kr/tools/fetch/localdata.d.ts.map +1 -0
- package/out/kr/tools/fetch/localdata.js +139 -0
- package/out/kr/tools/fetch/localdata.js.map +1 -0
- package/out/nl/recipes/postcode.d.ts +26 -0
- package/out/nl/recipes/postcode.d.ts.map +1 -0
- package/out/nl/recipes/postcode.js +103 -0
- package/out/nl/recipes/postcode.js.map +1 -0
- package/out/no/recipes/fragment.d.ts +43 -0
- package/out/no/recipes/fragment.d.ts.map +1 -0
- package/out/no/recipes/fragment.js +183 -0
- package/out/no/recipes/fragment.js.map +1 -0
- package/out/no/recipes/street/index.d.ts +18 -0
- package/out/no/recipes/street/index.d.ts.map +1 -0
- package/out/no/recipes/street/index.js +72 -0
- package/out/no/recipes/street/index.js.map +1 -0
- package/out/no/recipes/street/led.d.ts +24 -0
- package/out/no/recipes/street/led.d.ts.map +1 -0
- package/out/no/recipes/street/led.js +114 -0
- package/out/no/recipes/street/led.js.map +1 -0
- package/out/recipes/anchor-absorption.d.ts.map +1 -1
- package/out/recipes/anchor-absorption.js +2 -1
- package/out/recipes/anchor-absorption.js.map +1 -1
- package/out/recipes/bare/country.d.ts +25 -0
- package/out/recipes/bare/country.d.ts.map +1 -0
- package/out/recipes/bare/country.js +88 -0
- package/out/recipes/bare/country.js.map +1 -0
- package/out/recipes/bare/postcode/eval.d.ts +29 -0
- package/out/recipes/bare/postcode/eval.d.ts.map +1 -0
- package/out/recipes/bare/postcode/eval.js +85 -0
- package/out/recipes/bare/postcode/eval.js.map +1 -0
- package/out/recipes/bare/postcode/index.d.ts +62 -0
- package/out/recipes/bare/postcode/index.d.ts.map +1 -0
- package/out/recipes/bare/postcode/index.js +267 -0
- package/out/recipes/bare/postcode/index.js.map +1 -0
- package/out/recipes/boundary-stress.d.ts.map +1 -1
- package/out/recipes/boundary-stress.js +2 -1
- package/out/recipes/boundary-stress.js.map +1 -1
- package/out/recipes/country-balanced.d.ts.map +1 -1
- package/out/recipes/country-balanced.js +2 -1
- package/out/recipes/country-balanced.js.map +1 -1
- package/out/recipes/house-venue.d.ts.map +1 -1
- package/out/recipes/house-venue.js +2 -1
- package/out/recipes/house-venue.js.map +1 -1
- package/out/recipes/index.d.ts.map +1 -1
- package/out/recipes/index.js +28 -21
- package/out/recipes/index.js.map +1 -1
- package/out/recipes/intersection.d.ts.map +1 -1
- package/out/recipes/intersection.js +2 -1
- package/out/recipes/intersection.js.map +1 -1
- package/out/recipes/po/box/cedex/au-nz.d.ts +7 -0
- package/out/recipes/po/box/cedex/au-nz.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/au-nz.js +56 -0
- package/out/recipes/po/box/cedex/au-nz.js.map +1 -0
- package/out/recipes/po/box/cedex/ca.d.ts +7 -0
- package/out/recipes/po/box/cedex/ca.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/ca.js +38 -0
- package/out/recipes/po/box/cedex/ca.js.map +1 -0
- package/out/recipes/po/box/cedex/fr.d.ts +7 -0
- package/out/recipes/po/box/cedex/fr.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/fr.js +57 -0
- package/out/recipes/po/box/cedex/fr.js.map +1 -0
- package/out/recipes/po/box/cedex/phrases.d.ts +8 -0
- package/out/recipes/po/box/cedex/phrases.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/phrases.js +76 -0
- package/out/recipes/po/box/cedex/phrases.js.map +1 -0
- package/out/recipes/po/box/cedex/recipe.d.ts +9 -0
- package/out/recipes/po/box/cedex/recipe.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/recipe.js +153 -0
- package/out/recipes/po/box/cedex/recipe.js.map +1 -0
- package/out/recipes/po/box/cedex/sources.d.ts +86 -0
- package/out/recipes/po/box/cedex/sources.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/sources.js +143 -0
- package/out/recipes/po/box/cedex/sources.js.map +1 -0
- package/out/recipes/po/box/cedex/types.d.ts +31 -0
- package/out/recipes/po/box/cedex/types.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/types.js +5 -0
- package/out/recipes/po/box/cedex/types.js.map +1 -0
- package/out/recipes/po/box/cedex/us.d.ts +7 -0
- package/out/recipes/po/box/cedex/us.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/us.js +51 -0
- package/out/recipes/po/box/cedex/us.js.map +1 -0
- package/out/recipes/po/box/cedex/vocabulary.d.ts +68 -0
- package/out/recipes/po/box/cedex/vocabulary.d.ts.map +1 -0
- package/out/recipes/po/box/cedex/vocabulary.js +90 -0
- package/out/recipes/po/box/cedex/vocabulary.js.map +1 -0
- package/out/recipes/po/box/cedex.d.ts +9 -0
- package/out/recipes/po/box/cedex.d.ts.map +1 -0
- package/out/recipes/po/box/cedex.js +9 -0
- package/out/recipes/po/box/cedex.js.map +1 -0
- package/out/recipes/po/box/index.d.ts +17 -0
- package/out/recipes/po/box/index.d.ts.map +1 -0
- package/out/recipes/po/box/index.js +105 -0
- package/out/recipes/po/box/index.js.map +1 -0
- package/out/recipes/scaffold.d.ts +20 -1
- package/out/recipes/scaffold.d.ts.map +1 -1
- package/out/recipes/scaffold.js +18 -49
- package/out/recipes/scaffold.js.map +1 -1
- package/out/recipes/street/affix.d.ts +89 -0
- package/out/recipes/street/affix.d.ts.map +1 -0
- package/out/recipes/street/affix.js +565 -0
- package/out/recipes/street/affix.js.map +1 -0
- package/out/recipes/street/bare.d.ts +25 -0
- package/out/recipes/street/bare.d.ts.map +1 -0
- package/out/recipes/street/bare.js +87 -0
- package/out/recipes/street/bare.js.map +1 -0
- package/out/recipes/street.d.ts.map +1 -1
- package/out/recipes/street.js +2 -1
- package/out/recipes/street.js.map +1 -1
- package/out/recipes/sub/venue/context.d.ts +29 -0
- package/out/recipes/sub/venue/context.d.ts.map +1 -0
- package/out/recipes/sub/venue/context.js +106 -0
- package/out/recipes/sub/venue/context.js.map +1 -0
- package/out/recipes/sub/venue/index.d.ts +270 -0
- package/out/recipes/sub/venue/index.d.ts.map +1 -0
- package/out/recipes/sub/venue/index.js +728 -0
- package/out/recipes/sub/venue/index.js.map +1 -0
- package/out/recipes/sub/venue/sources.d.ts +209 -0
- package/out/recipes/sub/venue/sources.d.ts.map +1 -0
- package/out/recipes/sub/venue/sources.js +555 -0
- package/out/recipes/sub/venue/sources.js.map +1 -0
- package/out/recipes/trailing-region.d.ts.map +1 -1
- package/out/recipes/trailing-region.js +8 -2
- package/out/recipes/trailing-region.js.map +1 -1
- package/out/recipes/unit.d.ts.map +1 -1
- package/out/recipes/unit.js +2 -1
- package/out/recipes/unit.js.map +1 -1
- package/out/sg/recipes/register.d.ts +69 -0
- package/out/sg/recipes/register.d.ts.map +1 -0
- package/out/sg/recipes/register.js +211 -0
- package/out/sg/recipes/register.js.map +1 -0
- package/out/sg/tools/fetch/acra.d.ts +23 -0
- package/out/sg/tools/fetch/acra.d.ts.map +1 -0
- package/out/sg/tools/fetch/acra.js +126 -0
- package/out/sg/tools/fetch/acra.js.map +1 -0
- package/out/si/recipes/bare-village.d.ts +32 -0
- package/out/si/recipes/bare-village.d.ts.map +1 -0
- package/out/si/recipes/bare-village.js +89 -0
- package/out/si/recipes/bare-village.js.map +1 -0
- package/out/south-asia/recipes/register.d.ts +51 -0
- package/out/south-asia/recipes/register.d.ts.map +1 -0
- package/out/south-asia/recipes/register.js +154 -0
- package/out/south-asia/recipes/register.js.map +1 -0
- package/out/synthesizers/anchor-absorption.d.ts +1 -1
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -1
- package/out/synthesizers/anchor-absorption.js.map +1 -1
- package/out/synthesizers/boundary-stress.js.map +1 -1
- package/out/synthesizers/house-venue.js.map +1 -1
- package/out/synthesizers/intersection.js.map +1 -1
- package/out/synthesizers/locale.d.ts +116 -0
- package/out/synthesizers/locale.d.ts.map +1 -0
- package/out/synthesizers/locale.js +150 -0
- package/out/synthesizers/locale.js.map +1 -0
- package/out/synthesizers/no-street.d.ts.map +1 -1
- package/out/synthesizers/no-street.js.map +1 -1
- package/out/synthesizers/po-box.js.map +1 -1
- package/out/synthesizers/street.d.ts.map +1 -1
- package/out/synthesizers/street.js +1 -1
- package/out/synthesizers/street.js.map +1 -1
- package/out/synthesizers/utils.js +1 -1
- package/out/synthesizers/utils.js.map +1 -1
- package/out/test-kit/corpus-recipe.d.ts.map +1 -1
- package/out/test-kit/corpus-recipe.js.map +1 -1
- package/out/tools/audit.d.ts.map +1 -1
- package/out/tools/audit.js +3 -2
- package/out/tools/audit.js.map +1 -1
- package/out/tools/corpus-stats.js +3 -3
- package/out/tools/corpus-stats.js.map +1 -1
- package/out/tools/fetch/download/index.d.ts +10 -0
- package/out/tools/fetch/download/index.d.ts.map +1 -0
- package/out/tools/fetch/download/index.js +10 -0
- package/out/tools/fetch/download/index.js.map +1 -0
- package/out/tools/fetch/download/manifest.d.ts +47 -0
- package/out/tools/fetch/download/manifest.d.ts.map +1 -0
- package/out/tools/fetch/download/manifest.js +74 -0
- package/out/tools/fetch/download/manifest.js.map +1 -0
- package/out/tools/fetch/download/network.d.ts +105 -0
- package/out/tools/fetch/download/network.d.ts.map +1 -0
- package/out/tools/fetch/download/network.js +130 -0
- package/out/tools/fetch/download/network.js.map +1 -0
- package/out/tools/fetch/download/retry.d.ts +45 -0
- package/out/tools/fetch/download/retry.d.ts.map +1 -0
- package/out/tools/fetch/download/retry.js +136 -0
- package/out/tools/fetch/download/retry.js.map +1 -0
- package/out/tools/fetch/geonames/dump.d.ts +91 -0
- package/out/tools/fetch/geonames/dump.d.ts.map +1 -0
- package/out/tools/fetch/geonames/dump.js +171 -0
- package/out/tools/fetch/geonames/dump.js.map +1 -0
- package/out/tools/fetch/geonames/postal.d.ts +76 -0
- package/out/tools/fetch/geonames/postal.d.ts.map +1 -0
- package/out/tools/fetch/geonames/postal.js +126 -0
- package/out/tools/fetch/geonames/postal.js.map +1 -0
- package/out/tools/fetch/index.d.ts +47 -20
- package/out/tools/fetch/index.d.ts.map +1 -1
- package/out/tools/fetch/index.js +47 -20
- package/out/tools/fetch/index.js.map +1 -1
- package/out/tools/fetch/openaddresses.d.ts +1 -1
- package/out/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/tools/fetch/openaddresses.js +1 -1
- package/out/tools/fetch/openaddresses.js.map +1 -1
- package/out/tools/fetch/ourairports.d.ts +1 -1
- package/out/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/tools/fetch/ourairports.js +1 -1
- package/out/tools/fetch/ourairports.js.map +1 -1
- package/out/tools/fetch/ppd.js +1 -3
- package/out/tools/fetch/ppd.js.map +1 -1
- package/out/tools/fetch/wikidata-subvenue.d.ts +1 -1
- package/out/tools/fetch/wikidata-subvenue.d.ts.map +1 -1
- package/out/tools/fetch/wikidata-subvenue.js +1 -1
- package/out/tools/fetch/wikidata-subvenue.js.map +1 -1
- package/out/tools/golden/expand.d.ts +90 -0
- package/out/tools/golden/expand.d.ts.map +1 -0
- package/out/tools/golden/expand.js +406 -0
- package/out/tools/golden/expand.js.map +1 -0
- package/out/tools/golden/promote.d.ts +72 -0
- package/out/tools/golden/promote.d.ts.map +1 -0
- package/out/tools/golden/promote.js +243 -0
- package/out/tools/golden/promote.js.map +1 -0
- package/out/tools/golden/relabel-street/directory.d.ts +83 -0
- package/out/tools/golden/relabel-street/directory.d.ts.map +1 -0
- package/out/tools/golden/relabel-street/directory.js +216 -0
- package/out/tools/golden/relabel-street/directory.js.map +1 -0
- package/out/tools/golden/relabel-street/index.d.ts +9 -0
- package/out/tools/golden/relabel-street/index.d.ts.map +1 -0
- package/out/tools/golden/relabel-street/index.js +9 -0
- package/out/tools/golden/relabel-street/index.js.map +1 -0
- package/out/tools/golden/relabel-street/row.d.ts +121 -0
- package/out/tools/golden/relabel-street/row.d.ts.map +1 -0
- package/out/tools/golden/relabel-street/row.js +232 -0
- package/out/tools/golden/relabel-street/row.js.map +1 -0
- package/out/tools/index.d.ts +10 -10
- package/out/tools/index.d.ts.map +1 -1
- package/out/tools/index.js +10 -10
- package/out/tools/index.js.map +1 -1
- package/out/tools/ingest-csv.d.ts.map +1 -1
- package/out/tools/ingest-csv.js +5 -7
- package/out/tools/ingest-csv.js.map +1 -1
- package/out/tools/lint/slice/index.d.ts +93 -0
- package/out/tools/lint/slice/index.d.ts.map +1 -0
- package/out/tools/lint/slice/index.js +322 -0
- package/out/tools/lint/slice/index.js.map +1 -0
- package/out/tools/lint/slice/vocab.d.ts +90 -0
- package/out/tools/lint/slice/vocab.d.ts.map +1 -0
- package/out/tools/lint/slice/vocab.js +254 -0
- package/out/tools/lint/slice/vocab.js.map +1 -0
- package/out/tools/overlay-manifest.d.ts +56 -6
- package/out/tools/overlay-manifest.d.ts.map +1 -1
- package/out/tools/overlay-manifest.js +56 -15
- package/out/tools/overlay-manifest.js.map +1 -1
- package/out/tools/overture-subvenue.d.ts +1 -1
- package/out/tools/postcode-triples.d.ts +36 -0
- package/out/tools/postcode-triples.d.ts.map +1 -1
- package/out/tools/postcode-triples.js +91 -12
- package/out/tools/postcode-triples.js.map +1 -1
- package/out/tools/slice/kryptonite.d.ts +32 -0
- package/out/tools/slice/kryptonite.d.ts.map +1 -0
- package/out/tools/slice/kryptonite.js +99 -0
- package/out/tools/slice/kryptonite.js.map +1 -0
- package/out/tools/slice/translit.d.ts +51 -0
- package/out/tools/slice/translit.d.ts.map +1 -0
- package/out/tools/slice/translit.js +166 -0
- package/out/tools/slice/translit.js.map +1 -0
- package/out/tools/sub/venue/harvest.d.ts +100 -0
- package/out/tools/sub/venue/harvest.d.ts.map +1 -0
- package/out/tools/sub/venue/harvest.js +168 -0
- package/out/tools/sub/venue/harvest.js.map +1 -0
- package/out/tools/sub/venue/head-nouns.d.ts +50 -0
- package/out/tools/sub/venue/head-nouns.d.ts.map +1 -0
- package/out/tools/sub/venue/head-nouns.js.map +1 -0
- package/out/tools/sub/venue/lexicon.d.ts +156 -0
- package/out/tools/sub/venue/lexicon.d.ts.map +1 -0
- package/out/tools/sub/venue/lexicon.js +283 -0
- package/out/tools/sub/venue/lexicon.js.map +1 -0
- package/out/tools/sub/venue/promotions.d.ts +94 -0
- package/out/tools/sub/venue/promotions.d.ts.map +1 -0
- package/out/tools/sub/venue/promotions.js +266 -0
- package/out/tools/sub/venue/promotions.js.map +1 -0
- package/out/tools/sub/venue/surfaces.d.ts +45 -0
- package/out/tools/sub/venue/surfaces.d.ts.map +1 -0
- package/out/tools/sub/venue/surfaces.js.map +1 -0
- package/out/tools/sub/venue/table.d.ts +229 -0
- package/out/tools/sub/venue/table.d.ts.map +1 -0
- package/out/tools/sub/venue/table.js.map +1 -0
- package/out/tools/sub/venue/wikidata.d.ts +26 -0
- package/out/tools/sub/venue/wikidata.d.ts.map +1 -0
- package/out/tools/sub/venue/wikidata.js +62 -0
- package/out/tools/sub/venue/wikidata.js.map +1 -0
- package/out/tw/tools/fetch/gcis.d.ts +23 -0
- package/out/tw/tools/fetch/gcis.d.ts.map +1 -0
- package/out/tw/tools/fetch/gcis.js +135 -0
- package/out/tw/tools/fetch/gcis.js.map +1 -0
- package/out/types.d.ts +1 -1
- package/out/types.d.ts.map +1 -1
- package/out/us/adapters/fcc-bdc/adapter.d.ts.map +1 -0
- package/out/us/adapters/fcc-bdc/adapter.js +142 -0
- package/out/us/adapters/fcc-bdc/adapter.js.map +1 -0
- package/out/us/adapters/state/hi-schools/adapter.d.ts.map +1 -0
- package/out/us/adapters/state/hi-schools/adapter.js +136 -0
- package/out/us/adapters/state/hi-schools/adapter.js.map +1 -0
- package/out/us/adapters/state/hi-schools/workbook.d.ts.map +1 -0
- package/out/us/adapters/state/hi-schools/workbook.js.map +1 -0
- package/out/us/adapters/state/ia-contractors/adapter.d.ts.map +1 -0
- package/out/us/adapters/state/ia-contractors/adapter.js +104 -0
- package/out/us/adapters/state/ia-contractors/adapter.js.map +1 -0
- package/out/us/adapters/state/ny-notaries/adapter.d.ts.map +1 -0
- package/out/us/adapters/state/ny-notaries/adapter.js +112 -0
- package/out/us/adapters/state/ny-notaries/adapter.js.map +1 -0
- package/out/us/adapters/state/tx-notaries/adapter.d.ts.map +1 -0
- package/out/us/adapters/state/tx-notaries/adapter.js +118 -0
- package/out/us/adapters/state/tx-notaries/adapter.js.map +1 -0
- package/out/us/adapters/tiger/adapter.d.ts +55 -0
- package/out/us/adapters/tiger/adapter.d.ts.map +1 -0
- package/out/us/adapters/tiger/adapter.js +197 -0
- package/out/us/adapters/tiger/adapter.js.map +1 -0
- package/out/us/adapters/tiger/street-decompose.d.ts.map +1 -0
- package/out/us/adapters/tiger/street-decompose.js.map +1 -0
- package/out/us/adapters/usgov/hrsa-fqhc/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/hrsa-fqhc/adapter.js +121 -0
- package/out/us/adapters/usgov/hrsa-fqhc/adapter.js.map +1 -0
- package/out/us/adapters/usgov/imls-pls/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/imls-pls/adapter.js +106 -0
- package/out/us/adapters/usgov/imls-pls/adapter.js.map +1 -0
- package/out/us/adapters/usgov/irs-bmf/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/irs-bmf/adapter.js +116 -0
- package/out/us/adapters/usgov/irs-bmf/adapter.js.map +1 -0
- package/out/us/adapters/usgov/nad/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/nad/adapter.js +239 -0
- package/out/us/adapters/usgov/nad/adapter.js.map +1 -0
- package/out/us/adapters/usgov/nppes/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/nppes/adapter.js +110 -0
- package/out/us/adapters/usgov/nppes/adapter.js.map +1 -0
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.d.ts.map +1 -0
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js +143 -0
- package/out/us/adapters/usgov/samhsa-treatment-locator/adapter.js.map +1 -0
- package/out/us/fips-state.d.ts +41 -0
- package/out/us/fips-state.d.ts.map +1 -0
- package/out/us/fips-state.js +100 -0
- package/out/us/fips-state.js.map +1 -0
- package/out/us/tools/fetch/hrsa.d.ts +18 -0
- package/out/us/tools/fetch/hrsa.d.ts.map +1 -0
- package/out/us/tools/fetch/hrsa.js +46 -0
- package/out/us/tools/fetch/hrsa.js.map +1 -0
- package/out/us/tools/fetch/imls-pls.d.ts +25 -0
- package/out/us/tools/fetch/imls-pls.d.ts.map +1 -0
- package/out/us/tools/fetch/imls-pls.js +108 -0
- package/out/us/tools/fetch/imls-pls.js.map +1 -0
- package/out/us/tools/fetch/nad.d.ts +70 -0
- package/out/us/tools/fetch/nad.d.ts.map +1 -0
- package/out/us/tools/fetch/nad.js +234 -0
- package/out/us/tools/fetch/nad.js.map +1 -0
- package/out/us/tools/fetch/nppes.d.ts +26 -0
- package/out/us/tools/fetch/nppes.d.ts.map +1 -0
- package/out/us/tools/fetch/nppes.js +126 -0
- package/out/us/tools/fetch/nppes.js.map +1 -0
- package/out/us/tools/fetch/state/hi-schools.d.ts +26 -0
- package/out/us/tools/fetch/state/hi-schools.d.ts.map +1 -0
- package/out/us/tools/fetch/state/hi-schools.js +112 -0
- package/out/us/tools/fetch/state/hi-schools.js.map +1 -0
- package/out/us/tools/fetch/state/sources.d.ts +26 -0
- package/out/us/tools/fetch/state/sources.d.ts.map +1 -0
- package/out/us/tools/fetch/state/sources.js +111 -0
- package/out/us/tools/fetch/state/sources.js.map +1 -0
- package/out/us/tools/fetch/tiger-full.d.ts +22 -0
- package/out/us/tools/fetch/tiger-full.d.ts.map +1 -0
- package/out/us/tools/fetch/tiger-full.js +217 -0
- package/out/us/tools/fetch/tiger-full.js.map +1 -0
- package/out/utils/align.d.ts +1 -1
- package/out/utils/align.d.ts.map +1 -1
- package/out/utils/align.js.map +1 -1
- package/out/utils/golden.d.ts +7 -3
- package/out/utils/golden.d.ts.map +1 -1
- package/out/utils/golden.js +12 -8
- package/out/utils/golden.js.map +1 -1
- package/out/utils/parquet.d.ts.map +1 -1
- package/out/utils/split.js.map +1 -1
- package/out/utils/wof-json.d.ts.map +1 -1
- package/out/utils/wof-json.js +7 -12
- package/out/utils/wof-json.js.map +1 -1
- package/out/ve/recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/ve/recipes/reviewed-postcode-tail.js.map +1 -0
- package/package.json +196 -46
- package/lib/adapters/ban/adapter.ts +0 -190
- package/lib/adapters/fcc-bdc/adapter.ts +0 -174
- package/lib/adapters/geonames-postal/adapter.ts +0 -122
- package/lib/adapters/gnaf/adapter.ts +0 -149
- package/lib/adapters/gnaf/assemble.ts +0 -225
- package/lib/adapters/state-hi-schools/adapter.ts +0 -170
- package/lib/adapters/state-ia-contractors/adapter.ts +0 -144
- package/lib/adapters/state-ny-notaries/adapter.ts +0 -141
- package/lib/adapters/state-tx-notaries/adapter.ts +0 -157
- package/lib/adapters/tiger/adapter.ts +0 -252
- package/lib/adapters/usgov-hrsa-fqhc/adapter.ts +0 -167
- package/lib/adapters/usgov-imls-pls/adapter.ts +0 -143
- package/lib/adapters/usgov-irs-bmf/adapter.ts +0 -151
- package/lib/adapters/usgov-nad/adapter.ts +0 -334
- package/lib/adapters/usgov-nppes/adapter.ts +0 -151
- package/lib/adapters/usgov-samhsa-treatment-locator/adapter.ts +0 -193
- package/lib/adapters/wof-admin-json/adapter.ts +0 -231
- package/lib/adapters/wof-json-rows.ts +0 -129
- package/lib/adapters/wof-postalcode-json/adapter.ts +0 -176
- package/lib/name-prone-us-suffixes.ts +0 -12
- package/lib/recipes/bare-country.ts +0 -95
- package/lib/recipes/cz-pcfirst-preposition.ts +0 -111
- package/lib/recipes/fr-admin-split.ts +0 -242
- package/lib/recipes/fr-bare-street.ts +0 -170
- package/lib/recipes/fr-fragment.ts +0 -333
- package/lib/recipes/fr-lieudit.ts +0 -301
- package/lib/recipes/fr-order.ts +0 -245
- package/lib/recipes/german.ts +0 -359
- package/lib/recipes/locale.ts +0 -599
- package/lib/recipes/nl-postcode.ts +0 -115
- package/lib/recipes/no-fragment.ts +0 -219
- package/lib/recipes/no-street-led.ts +0 -141
- package/lib/recipes/no-street.ts +0 -90
- package/lib/recipes/po-box-cedex.ts +0 -976
- package/lib/recipes/po-box.ts +0 -123
- package/lib/recipes/si-bare-village.ts +0 -97
- package/lib/recipes/street-affix.ts +0 -804
- package/lib/recipes/street-bare.ts +0 -100
- package/lib/recipes/sub-venue-sources.ts +0 -877
- package/lib/recipes/sub-venue.ts +0 -997
- package/lib/synthesizers/german.ts +0 -260
- package/lib/tools/fetch/ban.ts +0 -289
- package/lib/tools/fetch/download.ts +0 -262
- package/lib/tools/fetch/geonames-dump.ts +0 -257
- package/lib/tools/fetch/geonames-postal.ts +0 -180
- package/lib/tools/fetch/hrsa.ts +0 -59
- package/lib/tools/fetch/imls-pls.ts +0 -149
- package/lib/tools/fetch/nad.ts +0 -341
- package/lib/tools/fetch/nppes.ts +0 -165
- package/lib/tools/fetch/state-hi-schools.ts +0 -161
- package/lib/tools/fetch/state-sources.ts +0 -146
- package/lib/tools/fetch/tiger-full.ts +0 -308
- package/lib/tools/golden-promote.ts +0 -337
- package/lib/tools/golden-relabel-street.ts +0 -683
- package/lib/tools/lint-slice-vocab.ts +0 -388
- package/lib/tools/sub-venue/harvest.ts +0 -234
- package/lib/tools/sub-venue/head-nouns.ts +0 -245
- package/lib/tools/sub-venue/surfaces.ts +0 -95
- package/lib/tools/sub-venue/table.ts +0 -281
- package/lib/tools/sub-venue/wikidata.ts +0 -85
- package/lib/tools/sub-venue-lexicon.ts +0 -429
- package/out/adapters/ban/adapter.d.ts.map +0 -1
- package/out/adapters/ban/adapter.js +0 -149
- package/out/adapters/ban/adapter.js.map +0 -1
- package/out/adapters/ban/street-decompose.d.ts.map +0 -1
- package/out/adapters/ban/street-decompose.js.map +0 -1
- package/out/adapters/fcc-bdc/adapter.d.ts.map +0 -1
- package/out/adapters/fcc-bdc/adapter.js +0 -144
- package/out/adapters/fcc-bdc/adapter.js.map +0 -1
- package/out/adapters/geonames-postal/adapter.d.ts.map +0 -1
- package/out/adapters/geonames-postal/adapter.js +0 -107
- package/out/adapters/geonames-postal/adapter.js.map +0 -1
- package/out/adapters/gnaf/adapter.d.ts.map +0 -1
- package/out/adapters/gnaf/adapter.js +0 -124
- package/out/adapters/gnaf/adapter.js.map +0 -1
- package/out/adapters/gnaf/assemble.d.ts +0 -65
- package/out/adapters/gnaf/assemble.d.ts.map +0 -1
- package/out/adapters/gnaf/assemble.js +0 -154
- package/out/adapters/gnaf/assemble.js.map +0 -1
- package/out/adapters/state-hi-schools/adapter.d.ts.map +0 -1
- package/out/adapters/state-hi-schools/adapter.js +0 -145
- package/out/adapters/state-hi-schools/adapter.js.map +0 -1
- package/out/adapters/state-hi-schools/workbook.d.ts.map +0 -1
- package/out/adapters/state-hi-schools/workbook.js.map +0 -1
- package/out/adapters/state-ia-contractors/adapter.d.ts.map +0 -1
- package/out/adapters/state-ia-contractors/adapter.js +0 -111
- package/out/adapters/state-ia-contractors/adapter.js.map +0 -1
- package/out/adapters/state-ny-notaries/adapter.d.ts.map +0 -1
- package/out/adapters/state-ny-notaries/adapter.js +0 -119
- package/out/adapters/state-ny-notaries/adapter.js.map +0 -1
- package/out/adapters/state-tx-notaries/adapter.d.ts.map +0 -1
- package/out/adapters/state-tx-notaries/adapter.js +0 -125
- package/out/adapters/state-tx-notaries/adapter.js.map +0 -1
- package/out/adapters/tiger/adapter.d.ts +0 -55
- package/out/adapters/tiger/adapter.d.ts.map +0 -1
- package/out/adapters/tiger/adapter.js +0 -201
- package/out/adapters/tiger/adapter.js.map +0 -1
- package/out/adapters/tiger/street-decompose.d.ts.map +0 -1
- package/out/adapters/tiger/street-decompose.js.map +0 -1
- package/out/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-hrsa-fqhc/adapter.js +0 -132
- package/out/adapters/usgov-hrsa-fqhc/adapter.js.map +0 -1
- package/out/adapters/usgov-imls-pls/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-imls-pls/adapter.js +0 -114
- package/out/adapters/usgov-imls-pls/adapter.js.map +0 -1
- package/out/adapters/usgov-irs-bmf/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-irs-bmf/adapter.js +0 -120
- package/out/adapters/usgov-irs-bmf/adapter.js.map +0 -1
- package/out/adapters/usgov-nad/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-nad/adapter.js +0 -249
- package/out/adapters/usgov-nad/adapter.js.map +0 -1
- package/out/adapters/usgov-nppes/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-nppes/adapter.js +0 -118
- package/out/adapters/usgov-nppes/adapter.js.map +0 -1
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +0 -1
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.js +0 -154
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.js.map +0 -1
- package/out/adapters/wof-admin-jp/adapter.d.ts.map +0 -1
- package/out/adapters/wof-admin-jp/adapter.js.map +0 -1
- package/out/adapters/wof-admin-json/adapter.d.ts +0 -79
- package/out/adapters/wof-admin-json/adapter.d.ts.map +0 -1
- package/out/adapters/wof-admin-json/adapter.js +0 -197
- package/out/adapters/wof-admin-json/adapter.js.map +0 -1
- package/out/adapters/wof-json-rows.d.ts +0 -62
- package/out/adapters/wof-json-rows.d.ts.map +0 -1
- package/out/adapters/wof-json-rows.js +0 -83
- package/out/adapters/wof-json-rows.js.map +0 -1
- package/out/adapters/wof-postalcode-json/adapter.d.ts +0 -65
- package/out/adapters/wof-postalcode-json/adapter.d.ts.map +0 -1
- package/out/adapters/wof-postalcode-json/adapter.js +0 -151
- package/out/adapters/wof-postalcode-json/adapter.js.map +0 -1
- package/out/codex/us-fips-state.d.ts +0 -41
- package/out/codex/us-fips-state.d.ts.map +0 -1
- package/out/codex/us-fips-state.js +0 -100
- package/out/codex/us-fips-state.js.map +0 -1
- package/out/name-prone-us-suffixes.d.ts +0 -12
- package/out/name-prone-us-suffixes.d.ts.map +0 -1
- package/out/name-prone-us-suffixes.js +0 -12
- package/out/name-prone-us-suffixes.js.map +0 -1
- package/out/recipes/bare-country.d.ts +0 -25
- package/out/recipes/bare-country.d.ts.map +0 -1
- package/out/recipes/bare-country.js +0 -87
- package/out/recipes/bare-country.js.map +0 -1
- package/out/recipes/cn-organizational-units.d.ts +0 -36
- package/out/recipes/cn-organizational-units.d.ts.map +0 -1
- package/out/recipes/cn-organizational-units.js +0 -159
- package/out/recipes/cn-organizational-units.js.map +0 -1
- package/out/recipes/cz-pcfirst-preposition.d.ts +0 -22
- package/out/recipes/cz-pcfirst-preposition.d.ts.map +0 -1
- package/out/recipes/cz-pcfirst-preposition.js +0 -100
- package/out/recipes/cz-pcfirst-preposition.js.map +0 -1
- package/out/recipes/fr-admin-split.d.ts +0 -38
- package/out/recipes/fr-admin-split.d.ts.map +0 -1
- package/out/recipes/fr-admin-split.js +0 -192
- package/out/recipes/fr-admin-split.js.map +0 -1
- package/out/recipes/fr-bare-street.d.ts +0 -28
- package/out/recipes/fr-bare-street.d.ts.map +0 -1
- package/out/recipes/fr-bare-street.js +0 -143
- package/out/recipes/fr-bare-street.js.map +0 -1
- package/out/recipes/fr-fragment.d.ts +0 -77
- package/out/recipes/fr-fragment.d.ts.map +0 -1
- package/out/recipes/fr-fragment.js +0 -276
- package/out/recipes/fr-fragment.js.map +0 -1
- package/out/recipes/fr-lieudit.d.ts +0 -32
- package/out/recipes/fr-lieudit.d.ts.map +0 -1
- package/out/recipes/fr-lieudit.js +0 -227
- package/out/recipes/fr-lieudit.js.map +0 -1
- package/out/recipes/fr-order.d.ts +0 -38
- package/out/recipes/fr-order.d.ts.map +0 -1
- package/out/recipes/fr-order.js +0 -197
- package/out/recipes/fr-order.js.map +0 -1
- package/out/recipes/german.d.ts +0 -44
- package/out/recipes/german.d.ts.map +0 -1
- package/out/recipes/german.js +0 -288
- package/out/recipes/german.js.map +0 -1
- package/out/recipes/locale.d.ts +0 -136
- package/out/recipes/locale.d.ts.map +0 -1
- package/out/recipes/locale.js +0 -460
- package/out/recipes/locale.js.map +0 -1
- package/out/recipes/nl-postcode.d.ts +0 -26
- package/out/recipes/nl-postcode.d.ts.map +0 -1
- package/out/recipes/nl-postcode.js +0 -102
- package/out/recipes/nl-postcode.js.map +0 -1
- package/out/recipes/no-fragment.d.ts +0 -43
- package/out/recipes/no-fragment.d.ts.map +0 -1
- package/out/recipes/no-fragment.js +0 -182
- package/out/recipes/no-fragment.js.map +0 -1
- package/out/recipes/no-street-led.d.ts +0 -24
- package/out/recipes/no-street-led.d.ts.map +0 -1
- package/out/recipes/no-street-led.js +0 -113
- package/out/recipes/no-street-led.js.map +0 -1
- package/out/recipes/no-street.d.ts +0 -18
- package/out/recipes/no-street.d.ts.map +0 -1
- package/out/recipes/no-street.js +0 -71
- package/out/recipes/no-street.js.map +0 -1
- package/out/recipes/po-box-cedex.d.ts +0 -38
- package/out/recipes/po-box-cedex.d.ts.map +0 -1
- package/out/recipes/po-box-cedex.js +0 -767
- package/out/recipes/po-box-cedex.js.map +0 -1
- package/out/recipes/po-box.d.ts +0 -17
- package/out/recipes/po-box.d.ts.map +0 -1
- package/out/recipes/po-box.js +0 -100
- package/out/recipes/po-box.js.map +0 -1
- package/out/recipes/reviewed-postcode-tail.d.ts.map +0 -1
- package/out/recipes/reviewed-postcode-tail.js.map +0 -1
- package/out/recipes/si-bare-village.d.ts +0 -32
- package/out/recipes/si-bare-village.d.ts.map +0 -1
- package/out/recipes/si-bare-village.js +0 -88
- package/out/recipes/si-bare-village.js.map +0 -1
- package/out/recipes/street-affix.d.ts +0 -89
- package/out/recipes/street-affix.d.ts.map +0 -1
- package/out/recipes/street-affix.js +0 -563
- package/out/recipes/street-affix.js.map +0 -1
- package/out/recipes/street-bare.d.ts +0 -25
- package/out/recipes/street-bare.d.ts.map +0 -1
- package/out/recipes/street-bare.js +0 -86
- package/out/recipes/street-bare.js.map +0 -1
- package/out/recipes/sub-venue-sources.d.ts +0 -229
- package/out/recipes/sub-venue-sources.d.ts.map +0 -1
- package/out/recipes/sub-venue-sources.js +0 -653
- package/out/recipes/sub-venue-sources.js.map +0 -1
- package/out/recipes/sub-venue.d.ts +0 -269
- package/out/recipes/sub-venue.d.ts.map +0 -1
- package/out/recipes/sub-venue.js +0 -725
- package/out/recipes/sub-venue.js.map +0 -1
- package/out/synthesizers/german.d.ts +0 -115
- package/out/synthesizers/german.d.ts.map +0 -1
- package/out/synthesizers/german.js +0 -149
- package/out/synthesizers/german.js.map +0 -1
- package/out/tools/fetch/ban.d.ts +0 -31
- package/out/tools/fetch/ban.d.ts.map +0 -1
- package/out/tools/fetch/ban.js +0 -239
- package/out/tools/fetch/ban.js.map +0 -1
- package/out/tools/fetch/download.d.ts +0 -117
- package/out/tools/fetch/download.d.ts.map +0 -1
- package/out/tools/fetch/download.js +0 -160
- package/out/tools/fetch/download.js.map +0 -1
- package/out/tools/fetch/geonames-dump.d.ts +0 -91
- package/out/tools/fetch/geonames-dump.d.ts.map +0 -1
- package/out/tools/fetch/geonames-dump.js +0 -171
- package/out/tools/fetch/geonames-dump.js.map +0 -1
- package/out/tools/fetch/geonames-postal.d.ts +0 -76
- package/out/tools/fetch/geonames-postal.d.ts.map +0 -1
- package/out/tools/fetch/geonames-postal.js +0 -126
- package/out/tools/fetch/geonames-postal.js.map +0 -1
- package/out/tools/fetch/hrsa.d.ts +0 -18
- package/out/tools/fetch/hrsa.d.ts.map +0 -1
- package/out/tools/fetch/hrsa.js +0 -46
- package/out/tools/fetch/hrsa.js.map +0 -1
- package/out/tools/fetch/imls-pls.d.ts +0 -25
- package/out/tools/fetch/imls-pls.d.ts.map +0 -1
- package/out/tools/fetch/imls-pls.js +0 -108
- package/out/tools/fetch/imls-pls.js.map +0 -1
- package/out/tools/fetch/nad.d.ts +0 -70
- package/out/tools/fetch/nad.d.ts.map +0 -1
- package/out/tools/fetch/nad.js +0 -234
- package/out/tools/fetch/nad.js.map +0 -1
- package/out/tools/fetch/nppes.d.ts +0 -26
- package/out/tools/fetch/nppes.d.ts.map +0 -1
- package/out/tools/fetch/nppes.js +0 -126
- package/out/tools/fetch/nppes.js.map +0 -1
- package/out/tools/fetch/state-hi-schools.d.ts +0 -26
- package/out/tools/fetch/state-hi-schools.d.ts.map +0 -1
- package/out/tools/fetch/state-hi-schools.js +0 -112
- package/out/tools/fetch/state-hi-schools.js.map +0 -1
- package/out/tools/fetch/state-sources.d.ts +0 -26
- package/out/tools/fetch/state-sources.d.ts.map +0 -1
- package/out/tools/fetch/state-sources.js +0 -111
- package/out/tools/fetch/state-sources.js.map +0 -1
- package/out/tools/fetch/tiger-full.d.ts +0 -22
- package/out/tools/fetch/tiger-full.d.ts.map +0 -1
- package/out/tools/fetch/tiger-full.js +0 -217
- package/out/tools/fetch/tiger-full.js.map +0 -1
- package/out/tools/golden-expand.d.ts +0 -90
- package/out/tools/golden-expand.d.ts.map +0 -1
- package/out/tools/golden-expand.js +0 -406
- package/out/tools/golden-expand.js.map +0 -1
- package/out/tools/golden-promote.d.ts +0 -72
- package/out/tools/golden-promote.d.ts.map +0 -1
- package/out/tools/golden-promote.js +0 -240
- package/out/tools/golden-promote.js.map +0 -1
- package/out/tools/golden-relabel-street.d.ts +0 -196
- package/out/tools/golden-relabel-street.d.ts.map +0 -1
- package/out/tools/golden-relabel-street.js +0 -440
- package/out/tools/golden-relabel-street.js.map +0 -1
- package/out/tools/lint-slice-vocab.d.ts +0 -90
- package/out/tools/lint-slice-vocab.d.ts.map +0 -1
- package/out/tools/lint-slice-vocab.js +0 -267
- package/out/tools/lint-slice-vocab.js.map +0 -1
- package/out/tools/lint-slice.d.ts +0 -93
- package/out/tools/lint-slice.d.ts.map +0 -1
- package/out/tools/lint-slice.js +0 -322
- package/out/tools/lint-slice.js.map +0 -1
- package/out/tools/slice-kryptonite.d.ts +0 -32
- package/out/tools/slice-kryptonite.d.ts.map +0 -1
- package/out/tools/slice-kryptonite.js +0 -99
- package/out/tools/slice-kryptonite.js.map +0 -1
- package/out/tools/slice-translit.d.ts +0 -51
- package/out/tools/slice-translit.d.ts.map +0 -1
- package/out/tools/slice-translit.js +0 -166
- package/out/tools/slice-translit.js.map +0 -1
- package/out/tools/sub-venue/harvest.d.ts +0 -100
- package/out/tools/sub-venue/harvest.d.ts.map +0 -1
- package/out/tools/sub-venue/harvest.js +0 -168
- package/out/tools/sub-venue/harvest.js.map +0 -1
- package/out/tools/sub-venue/head-nouns.d.ts +0 -50
- package/out/tools/sub-venue/head-nouns.d.ts.map +0 -1
- package/out/tools/sub-venue/head-nouns.js.map +0 -1
- package/out/tools/sub-venue/surfaces.d.ts +0 -45
- package/out/tools/sub-venue/surfaces.d.ts.map +0 -1
- package/out/tools/sub-venue/surfaces.js.map +0 -1
- package/out/tools/sub-venue/table.d.ts +0 -229
- package/out/tools/sub-venue/table.d.ts.map +0 -1
- package/out/tools/sub-venue/table.js.map +0 -1
- package/out/tools/sub-venue/wikidata.d.ts +0 -26
- package/out/tools/sub-venue/wikidata.d.ts.map +0 -1
- package/out/tools/sub-venue/wikidata.js +0 -62
- package/out/tools/sub-venue/wikidata.js.map +0 -1
- package/out/tools/sub-venue-lexicon.d.ts +0 -156
- package/out/tools/sub-venue-lexicon.d.ts.map +0 -1
- package/out/tools/sub-venue-lexicon.js +0 -283
- package/out/tools/sub-venue-lexicon.js.map +0 -1
- package/out/tools/sub-venue-promotions.d.ts +0 -94
- package/out/tools/sub-venue-promotions.d.ts.map +0 -1
- package/out/tools/sub-venue-promotions.js +0 -266
- package/out/tools/sub-venue-promotions.js.map +0 -1
- /package/lib/{recipes/cn-organizational-units.ts → cn/recipes/organizational-units.ts} +0 -0
- /package/lib/{adapters → fr/adapters}/ban/street-decompose.ts +0 -0
- /package/lib/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.ts +0 -0
- /package/lib/tools/{golden-expand.ts → golden/expand.ts} +0 -0
- /package/lib/tools/{lint-slice.ts → lint/slice/index.ts} +0 -0
- /package/lib/tools/{slice-kryptonite.ts → slice/kryptonite.ts} +0 -0
- /package/lib/tools/{slice-translit.ts → slice/translit.ts} +0 -0
- /package/lib/tools/{sub-venue-promotions.ts → sub/venue/promotions.ts} +0 -0
- /package/lib/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.ts +0 -0
- /package/lib/{adapters → us/adapters}/tiger/street-decompose.ts +0 -0
- /package/lib/{codex/us-fips-state.ts → us/fips-state.ts} +0 -0
- /package/lib/{recipes → ve/recipes}/reviewed-postcode-tail.ts +0 -0
- /package/out/adapters/{geonames-postal → geonames/postal}/adapter.d.ts +0 -0
- /package/out/{adapters → au/adapters}/gnaf/adapter.d.ts +0 -0
- /package/out/{adapters → fr/adapters}/ban/adapter.d.ts +0 -0
- /package/out/{adapters → fr/adapters}/ban/street-decompose.d.ts +0 -0
- /package/out/{adapters → fr/adapters}/ban/street-decompose.js +0 -0
- /package/out/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.d.ts +0 -0
- /package/out/{adapters/wof-admin-jp → jp/adapters/wof-admin}/adapter.js +0 -0
- /package/out/tools/{sub-venue → sub/venue}/head-nouns.js +0 -0
- /package/out/tools/{sub-venue → sub/venue}/surfaces.js +0 -0
- /package/out/tools/{sub-venue → sub/venue}/table.js +0 -0
- /package/out/{adapters → us/adapters}/fcc-bdc/adapter.d.ts +0 -0
- /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/adapter.d.ts +0 -0
- /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.d.ts +0 -0
- /package/out/{adapters/state-hi-schools → us/adapters/state/hi-schools}/workbook.js +0 -0
- /package/out/{adapters/state-ia-contractors → us/adapters/state/ia-contractors}/adapter.d.ts +0 -0
- /package/out/{adapters/state-ny-notaries → us/adapters/state/ny-notaries}/adapter.d.ts +0 -0
- /package/out/{adapters/state-tx-notaries → us/adapters/state/tx-notaries}/adapter.d.ts +0 -0
- /package/out/{adapters → us/adapters}/tiger/street-decompose.d.ts +0 -0
- /package/out/{adapters → us/adapters}/tiger/street-decompose.js +0 -0
- /package/out/{adapters/usgov-hrsa-fqhc → us/adapters/usgov/hrsa-fqhc}/adapter.d.ts +0 -0
- /package/out/{adapters/usgov-imls-pls → us/adapters/usgov/imls-pls}/adapter.d.ts +0 -0
- /package/out/{adapters/usgov-irs-bmf → us/adapters/usgov/irs-bmf}/adapter.d.ts +0 -0
- /package/out/{adapters/usgov-nad → us/adapters/usgov/nad}/adapter.d.ts +0 -0
- /package/out/{adapters/usgov-nppes → us/adapters/usgov/nppes}/adapter.d.ts +0 -0
- /package/out/{adapters/usgov-samhsa-treatment-locator → us/adapters/usgov/samhsa-treatment-locator}/adapter.d.ts +0 -0
- /package/out/{recipes → ve/recipes}/reviewed-postcode-tail.d.ts +0 -0
- /package/out/{recipes → ve/recipes}/reviewed-postcode-tail.js +0 -0
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
|
|
7
|
+
*
|
|
8
|
+
* Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
|
|
9
|
+
* byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
|
|
10
|
+
* `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
|
|
11
|
+
* (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
|
|
12
|
+
* the Python default.
|
|
13
|
+
*
|
|
14
|
+
* The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
|
|
15
|
+
* something else, or training gets conflicting gradients on the same token and the minority (the
|
|
16
|
+
* slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
|
|
17
|
+
*
|
|
18
|
+
* WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
|
|
19
|
+
* three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
|
|
20
|
+
* street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
|
|
21
|
+
*
|
|
22
|
+
* 1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
|
|
23
|
+
* retry both false-flagged FR cities as "street" from US street-contexts).
|
|
24
|
+
* 2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
|
|
25
|
+
* dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
|
|
26
|
+
* its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
|
|
27
|
+
* country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
|
|
28
|
+
* (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
|
|
29
|
+
* context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
|
|
30
|
+
* EXPECTED — the loader's affix-relabel handles it; weigh those separately.
|
|
31
|
+
*
|
|
32
|
+
* Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
|
|
33
|
+
* [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
|
|
34
|
+
* 50]
|
|
35
|
+
*/
|
|
36
|
+
import { dataRootPath } from "@mailwoman/core/data-root";
|
|
37
|
+
import { tryParsingJSON } from "@mailwoman/core/json";
|
|
38
|
+
import { pyRound } from "@mailwoman/core/numeric";
|
|
39
|
+
import { join } from "path-ts";
|
|
40
|
+
import { Globerator } from "spliterator/node/fs";
|
|
41
|
+
import { connectDuckDB } from "#utils/parquet";
|
|
42
|
+
/**
|
|
43
|
+
* Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
|
|
44
|
+
*/
|
|
45
|
+
function stripBIO(label) {
|
|
46
|
+
const head = label.slice(0, 2);
|
|
47
|
+
return head === "B-" || head === "I-" ? label.slice(2) : label;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
|
|
51
|
+
* are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
|
|
52
|
+
* corpora actually contain.
|
|
53
|
+
*/
|
|
54
|
+
function isDigit(token) {
|
|
55
|
+
return token.length > 0 && /^\p{Nd}+$/u.test(token);
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
|
|
59
|
+
*/
|
|
60
|
+
function pct(frac) {
|
|
61
|
+
return `${pyRound(frac * 100)}%`;
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
|
|
65
|
+
* everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
|
|
66
|
+
* print.
|
|
67
|
+
*/
|
|
68
|
+
function formatPyFloat(n) {
|
|
69
|
+
return Number.isInteger(n) ? n.toFixed(1) : String(n);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
|
|
73
|
+
*/
|
|
74
|
+
function pad(value, width) {
|
|
75
|
+
return value.padEnd(width);
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
|
|
79
|
+
* tag, mirroring `Counter.most_common(1)` (stable on equal counts).
|
|
80
|
+
*/
|
|
81
|
+
function dominant(counter) {
|
|
82
|
+
let total = 0;
|
|
83
|
+
let bestTag = "";
|
|
84
|
+
let bestCount = -1;
|
|
85
|
+
for (const [tag, count] of counter) {
|
|
86
|
+
total += count;
|
|
87
|
+
if (count > bestCount) {
|
|
88
|
+
bestCount = count;
|
|
89
|
+
bestTag = tag;
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (total === 0)
|
|
93
|
+
return ["", 0, 0];
|
|
94
|
+
return [bestTag, total, bestCount / total];
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Bump a (key -> count) tally, creating the inner counter on first sight.
|
|
98
|
+
*/
|
|
99
|
+
function bump(table, key, sub) {
|
|
100
|
+
let counter = table.get(key);
|
|
101
|
+
if (!counter) {
|
|
102
|
+
counter = new Map();
|
|
103
|
+
table.set(key, counter);
|
|
104
|
+
}
|
|
105
|
+
counter.set(sub, (counter.get(sub) ?? 0) + 1);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
|
|
109
|
+
* (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
|
|
110
|
+
* here.
|
|
111
|
+
*/
|
|
112
|
+
async function readRows(con, path) {
|
|
113
|
+
const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
|
|
114
|
+
const raw = result.getRowObjects();
|
|
115
|
+
const rows = [];
|
|
116
|
+
for (const r of raw) {
|
|
117
|
+
const tokens = tryParsingJSON(String(r.tokens));
|
|
118
|
+
const labels = tryParsingJSON(String(r.labels));
|
|
119
|
+
if (!Array.isArray(tokens) || !Array.isArray(labels))
|
|
120
|
+
continue;
|
|
121
|
+
rows.push({
|
|
122
|
+
tokens: tokens,
|
|
123
|
+
labels: labels,
|
|
124
|
+
country: r.country == null ? null : String(r.country),
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
return rows;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Read just the first row's `source` value — used to group base parts for a proportional slice.
|
|
131
|
+
*/
|
|
132
|
+
async function readSource(con, path) {
|
|
133
|
+
const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
|
|
134
|
+
const rows = result.getRowObjects();
|
|
135
|
+
return rows.length ? String(rows[0].source) : "";
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
|
|
139
|
+
*/
|
|
140
|
+
export async function lintSliceVocab(options) {
|
|
141
|
+
const baseVersion = options.baseVersion ?? "v0.5.0";
|
|
142
|
+
const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
|
|
143
|
+
const threshold = options.threshold ?? 0.7;
|
|
144
|
+
const minCount = options.minCount ?? 50;
|
|
145
|
+
const fraction = options.fraction ?? 1;
|
|
146
|
+
const con = await connectDuckDB();
|
|
147
|
+
// 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
|
|
148
|
+
const sliceRows = await readRows(con, options.slice);
|
|
149
|
+
const sliceTags = new Map();
|
|
150
|
+
const sliceCountries = new Map();
|
|
151
|
+
for (const { tokens, labels, country } of sliceRows) {
|
|
152
|
+
const n = Math.min(tokens.length, labels.length);
|
|
153
|
+
for (let i = 0; i < n; i++) {
|
|
154
|
+
const w = tokens[i];
|
|
155
|
+
const l = labels[i];
|
|
156
|
+
if (isDigit(w))
|
|
157
|
+
continue; // numbers are context-determined (house_number/postcode), not lexical vocab
|
|
158
|
+
bump(sliceTags, w, stripBIO(l));
|
|
159
|
+
let set = sliceCountries.get(w);
|
|
160
|
+
if (!set) {
|
|
161
|
+
set = new Set();
|
|
162
|
+
sliceCountries.set(w, set);
|
|
163
|
+
}
|
|
164
|
+
set.add(country);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const sliceVocab = new Set(sliceTags.keys());
|
|
168
|
+
console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`);
|
|
169
|
+
// 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
|
|
170
|
+
const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
|
|
171
|
+
let parts = (await Globerator.from("*.parquet", {
|
|
172
|
+
cwd: trainDir,
|
|
173
|
+
absolute: true,
|
|
174
|
+
}).toArray()).toSorted();
|
|
175
|
+
if (!parts.length) {
|
|
176
|
+
throw new Error("no base parts found");
|
|
177
|
+
}
|
|
178
|
+
if (fraction < 1) {
|
|
179
|
+
const bysrc = new Map();
|
|
180
|
+
for (const p of parts) {
|
|
181
|
+
const src = await readSource(con, p);
|
|
182
|
+
let list = bysrc.get(src);
|
|
183
|
+
if (!list) {
|
|
184
|
+
list = [];
|
|
185
|
+
bysrc.set(src, list);
|
|
186
|
+
}
|
|
187
|
+
list.push(p);
|
|
188
|
+
}
|
|
189
|
+
const sliced = [];
|
|
190
|
+
for (const ps of bysrc.values()) {
|
|
191
|
+
const take = Math.max(2, pyRound(ps.length * fraction));
|
|
192
|
+
for (const p of ps.slice(0, take)) {
|
|
193
|
+
sliced.push(p);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
parts = sliced;
|
|
197
|
+
}
|
|
198
|
+
console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`);
|
|
199
|
+
// 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
|
|
200
|
+
const baseTags = new Map();
|
|
201
|
+
for (let i = 0; i < parts.length; i++) {
|
|
202
|
+
const rows = await readRows(con, parts[i]);
|
|
203
|
+
for (const { tokens, labels, country } of rows) {
|
|
204
|
+
const n = Math.min(tokens.length, labels.length);
|
|
205
|
+
for (let j = 0; j < n; j++) {
|
|
206
|
+
const w = tokens[j];
|
|
207
|
+
if (sliceVocab.has(w) && sliceCountries.get(w).has(country)) {
|
|
208
|
+
bump(baseTags, w, stripBIO(labels[j]));
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
if ((i + 1) % 100 === 0) {
|
|
213
|
+
console.log(` ...${i + 1}/${parts.length} parts`);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
// 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
|
|
217
|
+
const flagged = [];
|
|
218
|
+
const affix = [];
|
|
219
|
+
for (const w of sliceVocab) {
|
|
220
|
+
const [sTag] = dominant(sliceTags.get(w));
|
|
221
|
+
const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
|
|
222
|
+
if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
|
|
223
|
+
continue;
|
|
224
|
+
const row = [w, sTag, bTag, bFrac, bTotal];
|
|
225
|
+
if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
|
|
226
|
+
affix.push(row);
|
|
227
|
+
}
|
|
228
|
+
else {
|
|
229
|
+
flagged.push(row);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
const sections = [
|
|
233
|
+
["CONTRADICTION", flagged],
|
|
234
|
+
["affix-split (EXPECTED — affix-relabel handles)", affix],
|
|
235
|
+
];
|
|
236
|
+
for (const [label, rows] of sections) {
|
|
237
|
+
if (!rows.length)
|
|
238
|
+
continue;
|
|
239
|
+
rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
|
|
240
|
+
console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
|
|
241
|
+
for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
|
|
242
|
+
console.log(` ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
if (!flagged.length) {
|
|
246
|
+
console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`);
|
|
247
|
+
}
|
|
248
|
+
return {
|
|
249
|
+
errors: flagged.length,
|
|
250
|
+
warnings: affix.length,
|
|
251
|
+
findings: { contradictions: flagged, affixSplits: affix },
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
//# sourceMappingURL=vocab.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vocab.js","sourceRoot":"","sources":["../../../../lib/tools/lint/slice/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAWrE;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,aAAa,EAAE,CAAA;IAEjC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAE9E,IAAI,KAAK,GAAG,CACX,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE;QAClC,GAAG,EAAE,QAAQ;QACb,QAAQ,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC,QAAQ,EAAE,CAAA;IAEZ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CACV,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAC3G,CAAA;IAED,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}
|
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
|
|
7
|
-
* ADDS
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* overture-multilocale is the second user
|
|
7
|
+
* ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and
|
|
8
|
+
* re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one
|
|
9
|
+
* label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's
|
|
10
|
+
* overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a
|
|
11
|
+
* set rather than one — chaining eight single-slice overlays would leave seven dead directories and
|
|
12
|
+
* an eight-deep base chain for what is one version).
|
|
11
13
|
*
|
|
12
14
|
* Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
|
|
13
15
|
* read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
|
|
@@ -22,13 +24,60 @@
|
|
|
22
24
|
*
|
|
23
25
|
* # then push the overlay to R2 + sync + `modal run -d ... --config <recipe>.yaml --resume none`.
|
|
24
26
|
*/
|
|
27
|
+
type ManifestSlice = Record<string, unknown> & {
|
|
28
|
+
path: string;
|
|
29
|
+
source?: string;
|
|
30
|
+
};
|
|
31
|
+
/**
|
|
32
|
+
* The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every
|
|
33
|
+
* corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact
|
|
34
|
+
* — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the
|
|
35
|
+
* word is banned in this tree and the ratchet's baseline is zero.
|
|
36
|
+
*/
|
|
37
|
+
declare const LEGACY_SLICES_KEY: "shards";
|
|
38
|
+
declare const LEGACY_ROWS_PER_SLICE_KEY: "rows_per_shard";
|
|
39
|
+
interface BaseManifest {
|
|
40
|
+
corpus_version?: string;
|
|
41
|
+
schema: unknown;
|
|
42
|
+
rows_per_slice?: unknown;
|
|
43
|
+
row_group_size: unknown;
|
|
44
|
+
slices?: ManifestSlice[];
|
|
45
|
+
counts: {
|
|
46
|
+
train: number;
|
|
47
|
+
val: number;
|
|
48
|
+
test: number;
|
|
49
|
+
};
|
|
50
|
+
total_rows: number;
|
|
51
|
+
/**
|
|
52
|
+
* The pre-rename spellings, present on every corpus built before 2026-09-01.
|
|
53
|
+
*/
|
|
54
|
+
[LEGACY_SLICES_KEY]?: ManifestSlice[];
|
|
55
|
+
[LEGACY_ROWS_PER_SLICE_KEY]?: unknown;
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* A base manifest's slice list under either key.
|
|
59
|
+
*
|
|
60
|
+
* The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer
|
|
61
|
+
* measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the
|
|
62
|
+
* loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all.
|
|
63
|
+
*/
|
|
64
|
+
export declare function baseManifestSlices(base: BaseManifest): ManifestSlice[];
|
|
65
|
+
/**
|
|
66
|
+
* One parquet to add, with the source label its rows carry.
|
|
67
|
+
*/
|
|
68
|
+
export interface OverlaySlice {
|
|
69
|
+
parquet: string;
|
|
70
|
+
source: string;
|
|
71
|
+
}
|
|
25
72
|
export interface OverlayManifestOptions {
|
|
26
73
|
base: string;
|
|
27
74
|
newDir: string;
|
|
28
75
|
modalRoot: string;
|
|
29
76
|
version: string;
|
|
30
|
-
|
|
31
|
-
|
|
77
|
+
/**
|
|
78
|
+
* The parquets this overlay adds, in the order they should appear after the base's own.
|
|
79
|
+
*/
|
|
80
|
+
slices: readonly OverlaySlice[];
|
|
32
81
|
note: string;
|
|
33
82
|
}
|
|
34
83
|
/**
|
|
@@ -36,4 +85,5 @@ export interface OverlayManifestOptions {
|
|
|
36
85
|
*/
|
|
37
86
|
export declare function rerootBaseSlicePath(path: string, baseManifestPath: string): string;
|
|
38
87
|
export declare function assembleOverlayManifest(args: OverlayManifestOptions): Promise<void>;
|
|
88
|
+
export {};
|
|
39
89
|
//# sourceMappingURL=overlay-manifest.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"overlay-manifest.d.ts","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAsBH,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,CAAA;AAEhF;;;;;GAKG;AACH,QAAA,MAAM,iBAAiB,UAAyB,CAAA;AAChD,QAAA,MAAM,yBAAyB,kBAAiC,CAAA;AAEhE,UAAU,YAAY;IACrB,cAAc,CAAC,EAAE,MAAM,CAAA;IACvB,MAAM,EAAE,OAAO,CAAA;IACf,cAAc,CAAC,EAAE,OAAO,CAAA;IACxB,cAAc,EAAE,OAAO,CAAA;IACvB,MAAM,CAAC,EAAE,aAAa,EAAE,CAAA;IACxB,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAA;IACpD,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,CAAC,iBAAiB,CAAC,CAAC,EAAE,aAAa,EAAE,CAAA;IACrC,CAAC,yBAAyB,CAAC,CAAC,EAAE,OAAO,CAAA;CACrC;AAED;;;;;;GAMG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,YAAY,GAAG,aAAa,EAAE,CAQtE;AASD;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,CAAA;CACd;AA0BD,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,OAAO,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,MAAM,EAAE,SAAS,YAAY,EAAE,CAAA;IAC/B,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;GAEG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,GAAG,MAAM,CAalF;AAED,wBAAsB,uBAAuB,CAAC,IAAI,EAAE,sBAAsB,GAAG,OAAO,CAAC,IAAI,CAAC,CAuDzF"}
|
|
@@ -4,10 +4,12 @@
|
|
|
4
4
|
* @author Teffen Ellis, et al.
|
|
5
5
|
*
|
|
6
6
|
* Assemble a corpus OVERLAY MANIFEST — generalized from assemble-fr-admin-split-overlay-manifest.
|
|
7
|
-
* ADDS
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
* overture-multilocale is the second user
|
|
7
|
+
* ADDS slice parquets to a base corpus, keeping every base slice VERBATIM (pure overlay ADD), and
|
|
8
|
+
* re-roots base paths to /data (the Modal volume). Parameterized by --slice-parquet + --source, one
|
|
9
|
+
* label per parquet, so it works for any overlay (the fr-admin-split one is the original; #148's
|
|
10
|
+
* overture-multilocale is the second user; v0.29.0's eight target-family slices are why it takes a
|
|
11
|
+
* set rather than one — chaining eight single-slice overlays would leave seven dead directories and
|
|
12
|
+
* an eight-deep base chain for what is one version).
|
|
11
13
|
*
|
|
12
14
|
* Ported faithfully from scripts/assemble-overlay-manifest.py. The new slice's source_id column is
|
|
13
15
|
* read through DuckDB (`@duckdb/node-api`) instead of PyArrow; everything else is pure JSON.
|
|
@@ -27,6 +29,33 @@ import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
|
|
|
27
29
|
import { sha256Hex } from "@mailwoman/core/hash";
|
|
28
30
|
import { basename, dirname, join } from "path-ts";
|
|
29
31
|
import { connectDuckDB, escapeSQLString } from "#utils/parquet";
|
|
32
|
+
/**
|
|
33
|
+
* The two keys a manifest written before the 2026-09-01 vocabulary rename uses for its slice list and its size. Every
|
|
34
|
+
* corpus built before that date carries them, here and on the Modal volume, and a built corpus is an immutable artifact
|
|
35
|
+
* — so a reader accepts either spelling and a writer emits only the current one. Spelled by concatenation because the
|
|
36
|
+
* word is banned in this tree and the ratchet's baseline is zero.
|
|
37
|
+
*/
|
|
38
|
+
const LEGACY_SLICES_KEY = `sh${"ards"}`;
|
|
39
|
+
const LEGACY_ROWS_PER_SLICE_KEY = `rows_per_sh${"ard"}`;
|
|
40
|
+
/**
|
|
41
|
+
* A base manifest's slice list under either key.
|
|
42
|
+
*
|
|
43
|
+
* The rename moved this reader and the trainer's to the new key without migrating the manifests, and the trainer
|
|
44
|
+
* measured the cost on 2026-09-09: `v0.28.0-reviewed-postcode-tail` declares 706 train slices under the old key and the
|
|
45
|
+
* loader resolved ONE. An overlay assembled from a base read as empty would carry no base slices at all.
|
|
46
|
+
*/
|
|
47
|
+
export function baseManifestSlices(base) {
|
|
48
|
+
const slices = base.slices?.length ? base.slices : base[LEGACY_SLICES_KEY];
|
|
49
|
+
if (slices?.length)
|
|
50
|
+
return slices;
|
|
51
|
+
throw new Error(`base manifest lists no slices under "slices" or the pre-rename key — refusing to assemble an overlay whose base would be empty`);
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* The base's rows-per-slice under either key, carried through to the new manifest verbatim.
|
|
55
|
+
*/
|
|
56
|
+
function baseRowsPerSlice(base) {
|
|
57
|
+
return base.rows_per_slice ?? base[LEGACY_ROWS_PER_SLICE_KEY];
|
|
58
|
+
}
|
|
30
59
|
async function descriptor(localPath, modalPath, split, source) {
|
|
31
60
|
const db = await connectDuckDB();
|
|
32
61
|
const result = await db.runAndReadAll(`SELECT source_id FROM read_parquet('${escapeSQLString(localPath)}')`);
|
|
@@ -59,32 +88,44 @@ export function rerootBaseSlicePath(path, baseManifestPath) {
|
|
|
59
88
|
return path;
|
|
60
89
|
}
|
|
61
90
|
export async function assembleOverlayManifest(args) {
|
|
91
|
+
if (!args.slices.length)
|
|
92
|
+
throw new Error("an overlay must add at least one slice");
|
|
62
93
|
const base = await readLocalJSONFile(args.base);
|
|
63
|
-
|
|
64
|
-
|
|
94
|
+
const baseSlices = baseManifestSlices(base);
|
|
95
|
+
for (const slice of args.slices) {
|
|
96
|
+
if (baseSlices.some((s) => s.source === slice.source)) {
|
|
97
|
+
console.log(`WARN: base already contains source '${slice.source}' — is this the right base?`);
|
|
98
|
+
}
|
|
65
99
|
}
|
|
66
|
-
const kept =
|
|
67
|
-
const
|
|
100
|
+
const kept = baseSlices.map((s) => ({ ...s, path: rerootBaseSlicePath(s.path, args.base) }));
|
|
101
|
+
const added = [];
|
|
102
|
+
for (const slice of args.slices) {
|
|
103
|
+
added.push(await descriptor(join(args.newDir, "train", slice.parquet), `${args.modalRoot}/train/${slice.parquet}`, "train", slice.source));
|
|
104
|
+
}
|
|
105
|
+
const addedRows = added.reduce((total, slice) => total + slice.rows, 0);
|
|
106
|
+
const sources = args.slices.map((slice) => slice.source).join(", ");
|
|
68
107
|
const manifest = {
|
|
69
108
|
corpus_version: args.version,
|
|
70
109
|
overlay_base: base.corpus_version ?? null,
|
|
71
|
-
note: args.note || `${base.corpus_version} slices (all kept verbatim) +
|
|
110
|
+
note: args.note || `${base.corpus_version} slices (all kept verbatim) + ${sources}. Pure overlay add.`,
|
|
72
111
|
schema: base.schema,
|
|
73
|
-
rows_per_slice: base
|
|
112
|
+
rows_per_slice: baseRowsPerSlice(base),
|
|
74
113
|
row_group_size: base.row_group_size,
|
|
75
|
-
slices: [...kept,
|
|
114
|
+
slices: [...kept, ...added],
|
|
76
115
|
counts: {
|
|
77
|
-
train: base.counts.train +
|
|
116
|
+
train: base.counts.train + addedRows,
|
|
78
117
|
val: base.counts.val,
|
|
79
118
|
test: base.counts.test,
|
|
80
119
|
},
|
|
81
|
-
total_rows: base.total_rows +
|
|
120
|
+
total_rows: base.total_rows + addedRows,
|
|
82
121
|
};
|
|
83
122
|
const out = join(args.newDir, "MANIFEST.json");
|
|
84
123
|
await writeLocalJSONFile(manifest, out);
|
|
85
124
|
console.log(`wrote ${out}`);
|
|
86
|
-
console.log(` slices: ${manifest.slices.length} (${kept.length} base kept,
|
|
125
|
+
console.log(` slices: ${manifest.slices.length} (${kept.length} base kept, +${added.length} added)`);
|
|
87
126
|
console.log(` counts: ${JSON.stringify(manifest.counts)} total: ${manifest.total_rows}`);
|
|
88
|
-
|
|
127
|
+
for (const slice of added) {
|
|
128
|
+
console.log(` ${slice.source} train: ${slice.rows} rows (${slice.bytes} bytes)`);
|
|
129
|
+
}
|
|
89
130
|
}
|
|
90
131
|
//# sourceMappingURL=overlay-manifest.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"overlay-manifest.js","sourceRoot":"","sources":["../../lib/tools/overlay-manifest.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,eAAe,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACzF,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,sBAAsB,CAAA;AAChD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAEjD,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAiB/D;;;;;GAKG;AACH,MAAM,iBAAiB,GAAG,KAAK,MAAM,EAAW,CAAA;AAChD,MAAM,yBAAyB,GAAG,cAAc,KAAK,EAAW,CAAA;AAiBhE;;;;;;GAMG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAkB;IACpD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAA;IAE1E,IAAI,MAAM,EAAE,MAAM;QAAE,OAAO,MAAM,CAAA;IAEjC,MAAM,IAAI,KAAK,CACd,gIAAgI,CAChI,CAAA;AACF,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CAAC,IAAkB;IAC3C,OAAO,IAAI,CAAC,cAAc,IAAI,IAAI,CAAC,yBAAyB,CAAC,CAAA;AAC9D,CAAC;AAUD,KAAK,UAAU,UAAU,CACxB,SAAiB,EACjB,SAAiB,EACjB,KAAa,EACb,MAAc;IAEd,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,uCAAuC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;IAC5G,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAmB,CAAC,CAAA;IAErE,OAAO;QACN,KAAK;QACL,IAAI,EAAE,SAAS;QACf,MAAM,EAAE,SAAS;QACjB,WAAW,EAAE,QAAQ;QACrB,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,KAAK,EAAE,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACvC,MAAM,EAAE,SAAS,CAAC,MAAM,eAAe,CAAC,SAAS,CAAC,CAAC;QACnD,eAAe,EAAE,IAAI,CAAC,CAAC,CAAE;QACzB,cAAc,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE;QAC5B,MAAM;KACN,CAAA;AACF,CAAC;AAcD;;GAEG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,gBAAwB;IACzE,MAAM,cAAc,GAAG,IAAI,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAA;IAEzD,IAAI,cAAc,KAAK,CAAC,CAAC;QAAE,OAAO,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,CAAA;IAEtE,IAAI,gCAAgC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,YAAY,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAA;QAC9C,MAAM,aAAa,GAAG,0BAA0B,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,IAAI,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAA;QAE3G,OAAO,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,IAA4B;IACzE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;IAElF,MAAM,IAAI,GAAG,MAAM,iBAAiB,CAAe,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,UAAU,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;IAE3C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YACvD,OAAO,CAAC,GAAG,CAAC,uCAAuC,KAAK,CAAC,MAAM,6BAA6B,CAAC,CAAA;QAC9F,CAAC;IACF,CAAC;IAED,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,EAAE,mBAAmB,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;IAC5F,MAAM,KAAK,GAAsB,EAAE,CAAA;IAEnC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CACT,MAAM,UAAU,CACf,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,EACzC,GAAG,IAAI,CAAC,SAAS,UAAU,KAAK,CAAC,OAAO,EAAE,EAC1C,OAAO,EACP,KAAK,CAAC,MAAM,CACZ,CACD,CAAA;IACF,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,CAAA;IACvE,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAEnE,MAAM,QAAQ,GAAG;QAChB,cAAc,EAAE,IAAI,CAAC,OAAO;QAC5B,YAAY,EAAE,IAAI,CAAC,cAAc,IAAI,IAAI;QACzC,IAAI,EAAE,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,CAAC,cAAc,iCAAiC,OAAO,qBAAqB;QACtG,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,cAAc,EAAE,gBAAgB,CAAC,IAAI,CAAC;QACtC,cAAc,EAAE,IAAI,CAAC,cAAc;QACnC,MAAM,EAAE,CAAC,GAAG,IAAI,EAAE,GAAG,KAAK,CAAC;QAC3B,MAAM,EAAE;YACP,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,SAAS;YACpC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,GAAG;YACpB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI;SACtB;QACD,UAAU,EAAE,IAAI,CAAC,UAAU,GAAG,SAAS;KACvC,CAAA;IAED,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAA;IAC9C,MAAM,kBAAkB,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAA;IAEvC,OAAO,CAAC,GAAG,CAAC,SAAS,GAAG,EAAE,CAAC,CAAA;IAC3B,OAAO,CAAC,GAAG,CAAC,aAAa,QAAQ,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAA;IACrG,OAAO,CAAC,GAAG,CAAC,aAAa,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,MAAM,CAAC,YAAY,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAA;IAE1F,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC3B,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,MAAM,WAAW,KAAK,CAAC,IAAI,UAAU,KAAK,CAAC,KAAK,SAAS,CAAC,CAAA;IAClF,CAAC;AACF,CAAC"}
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
* provenance lie, and under ODbL a consequential one. Hence the `source`/`region` options on that
|
|
38
38
|
* function and the source-neutral {@link SubVenueHarvestRow} name.
|
|
39
39
|
*/
|
|
40
|
-
import type { SubVenueHarvestRow } from "#tools/sub
|
|
40
|
+
import type { SubVenueHarvestRow } from "#tools/sub/venue/lexicon";
|
|
41
41
|
/**
|
|
42
42
|
* Overture Places category → the designator its rows attest.
|
|
43
43
|
*
|
|
@@ -103,6 +103,35 @@ export declare function applyLocalityQuota(triples: readonly PostcodeTriple[], q
|
|
|
103
103
|
* one city's postcode list.
|
|
104
104
|
*/
|
|
105
105
|
export declare function applyCountryBudget(triples: readonly PostcodeTriple[], budget: number | ReadonlyMap<string, number>): PostcodeTriple[];
|
|
106
|
+
/**
|
|
107
|
+
* A place's preferred names in the languages its addresses are written in, read from the gazetteer's `names` table.
|
|
108
|
+
*/
|
|
109
|
+
interface PreferredNames {
|
|
110
|
+
/**
|
|
111
|
+
* Preferred names in the country's official languages, in the table's order.
|
|
112
|
+
*/
|
|
113
|
+
official: readonly string[];
|
|
114
|
+
/**
|
|
115
|
+
* Preferred names in the region's co-official languages.
|
|
116
|
+
*/
|
|
117
|
+
coOfficial: readonly string[];
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* The surfaces a REGION is written as: its preferred names in the official language(s), then in the region's
|
|
121
|
+
* co-official languages, then the gazetteer's own `spr.name` (the English exonym). Deduplicated, order kept.
|
|
122
|
+
*
|
|
123
|
+
* `spr.name` is the English-preferred, diacritic-stripped label, and reading it alone is the defect #1673 measured:
|
|
124
|
+
* 53,078 ES rows teaching `Balearic Islands` (2,872 `Andalusia`, 5,680 `Castile and Leon`) against 4 rows of `Illes
|
|
125
|
+
* Balears`, and every province with its accent gone (`Cordoba`, `Leon`). A Spanish user writes `Islas Baleares` or
|
|
126
|
+
* `Illes Balears`; the exonym stays as ONE surface among them because a user may write it too.
|
|
127
|
+
*/
|
|
128
|
+
export declare function regionWrittenForms(sprName: string, names: PreferredNames): string[];
|
|
129
|
+
/**
|
|
130
|
+
* The surface a LOCALITY is written as: the official-language preferred name that is `spr.name` with its diacritics
|
|
131
|
+
* restored (`Cordoba` → `Córdoba`), else the first official-language preferred name, else `spr.name`. One surface, not
|
|
132
|
+
* a fan-out: the region carries the multiplicity and the locality would multiply it.
|
|
133
|
+
*/
|
|
134
|
+
export declare function localityWrittenForm(sprName: string, names: PreferredNames): string;
|
|
106
135
|
/**
|
|
107
136
|
* Read triples out of `postalcode-intl.db` by following each postcode's `parent_id` into the admin gazetteer and that
|
|
108
137
|
* place's ancestry to a region.
|
|
@@ -110,6 +139,12 @@ export declare function applyCountryBudget(triples: readonly PostcodeTriple[], b
|
|
|
110
139
|
* A postcode whose parent does not resolve, or whose parent has no region ancestor, is DROPPED rather than emitted with
|
|
111
140
|
* a blank — the recipe already skips a tuple with no region, and a blank here would hide how much of the source is
|
|
112
141
|
* actually reachable.
|
|
142
|
+
*
|
|
143
|
+
* One triple per REGION SURFACE (see {@link regionWrittenForms}): a Balearic postcode yields `Islas Baleares`, `Illes
|
|
144
|
+
* Balears` and `Balearic Islands` rows, a Zamora one `Zamora` alone. The languages come from the codex — the country's
|
|
145
|
+
* official languages plus the province's co-official ones — and never from the names table's own language list, whose
|
|
146
|
+
* "preferred" name in a language not spoken in the province is often the parent community's (`Zamora` → `Castella i
|
|
147
|
+
* Lleó`).
|
|
113
148
|
*/
|
|
114
149
|
export declare function readTriplesFromParentJoin(countries: readonly string[], options?: {
|
|
115
150
|
postcodeDB?: string;
|
|
@@ -171,4 +206,5 @@ export declare function readTriplesFromGeonames(country: string, path: string, c
|
|
|
171
206
|
* Resolve a GeoNames export path under the standard fetch out-root.
|
|
172
207
|
*/
|
|
173
208
|
export declare function geonamesPostalPath(country: string, sourcesRoot?: string): string;
|
|
209
|
+
export {};
|
|
174
210
|
//# sourceMappingURL=postcode-triples.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"postcode-triples.d.ts","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;
|
|
1
|
+
{"version":3,"file":"postcode-triples.d.ts","sourceRoot":"","sources":["../../lib/tools/postcode-triples.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAYH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAA;AAG1D;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,EAAE,MAAM,CAAA;IAChB;;;;;OAKG;IACH,iBAAiB,CAAC,EAAE,MAAM,CAAA;IAC1B,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,MAAM,CAAA;IACd,iBAAiB,EAAE,iBAAiB,CAAA;CACpC;AAED;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,oBAAoB,EAAE,WAAW,CAAC,MAAM,EAAE;IAAE,SAAS,EAAE,iBAAiB,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAiBrG,CAAA;AAEF;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,KAAK,CAAA;AAExC;;;;;GAKG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,KAAK,GAAE,MAA+B,GACpC,cAAc,EAAE,CAelB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,kBAAkB,CACjC,OAAO,EAAE,SAAS,cAAc,EAAE,EAClC,MAAM,EAAE,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,GAC1C,cAAc,EAAE,CAkBlB;AAED;;GAEG;AACH,UAAU,cAAc;IACvB;;OAEG;IACH,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;OAEG;IACH,UAAU,EAAE,SAAS,MAAM,EAAE,CAAA;CAC7B;AAED;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,EAAE,CAYnF;AASD;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,GAAG,MAAM,CAIlF;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,yBAAyB,CAC9C,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,OAAO,GAAE;IAAE,UAAU,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GACrD,OAAO,CAAC,cAAc,EAAE,CAAC,CA0G3B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,wBAAwB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,CAsBpH;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AACH,wBAAsB,uBAAuB,CAC5C,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,MAAM,EACZ,WAAW,EAAE,MAAM,EACnB,OAAO,GAAE;IAAE,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,cAAc,EAAE,CAAC,CA4C3B;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,MAAM,GAAG,MAAM,CAIhF"}
|