@mailwoman/corpus 9.2.0 → 9.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -22
- package/data/PROVENANCE.md +7 -7
- package/lib/adapters/ban/adapter.ts +190 -0
- package/lib/adapters/ban/street-decompose.ts +66 -0
- package/lib/adapters/fcc-bdc/adapter.ts +174 -0
- package/lib/adapters/geonames/adapter.ts +199 -0
- package/lib/adapters/geonames-postal/adapter.ts +122 -0
- package/lib/adapters/gnaf/adapter.ts +149 -0
- package/lib/adapters/gnaf/assemble.ts +225 -0
- package/lib/adapters/index.ts +153 -0
- package/lib/adapters/openaddresses/adapter.ts +251 -0
- package/lib/adapters/overture/adapter.ts +162 -0
- package/lib/adapters/state-hi-schools/adapter.ts +170 -0
- package/lib/adapters/state-hi-schools/workbook.ts +35 -0
- package/lib/adapters/synth-po-box/adapter.ts +204 -0
- package/lib/adapters/tiger/adapter.ts +252 -0
- package/lib/adapters/tiger/street-decompose.ts +83 -0
- package/lib/adapters/usgov-hrsa-fqhc/adapter.ts +167 -0
- package/lib/adapters/usgov-imls-pls/adapter.ts +143 -0
- package/lib/adapters/usgov-irs-bmf/adapter.ts +151 -0
- package/lib/adapters/usgov-nad/adapter.ts +334 -0
- package/lib/adapters/usgov-nppes/adapter.ts +151 -0
- package/lib/adapters/usgov-samhsa-treatment-locator/adapter.ts +193 -0
- package/lib/adapters/utils/index.ts +307 -0
- package/lib/adapters/wof-admin-jp/adapter.ts +214 -0
- package/lib/adapters/wof-admin-json/adapter.ts +231 -0
- package/lib/adapters/wof-json-rows.ts +129 -0
- package/lib/adapters/wof-postalcode-json/adapter.ts +176 -0
- package/lib/build.ts +378 -0
- package/lib/env.ts +50 -0
- package/lib/index.ts +13 -0
- package/lib/recipes/anchor-absorption.ts +101 -0
- package/lib/recipes/bare-country.ts +95 -0
- package/lib/recipes/boundary-stress.ts +102 -0
- package/lib/recipes/cn-organizational-units.ts +199 -0
- package/lib/recipes/country-balanced.ts +510 -0
- package/lib/recipes/cz-pcfirst-preposition.ts +112 -0
- package/lib/recipes/fr-admin-split.ts +243 -0
- package/lib/recipes/fr-bare-street.ts +171 -0
- package/lib/recipes/fr-fragment.ts +334 -0
- package/lib/recipes/fr-lieudit.ts +301 -0
- package/lib/recipes/fr-order.ts +246 -0
- package/lib/recipes/german.ts +360 -0
- package/lib/recipes/house-venue.ts +93 -0
- package/lib/recipes/index.ts +97 -0
- package/lib/recipes/intersection.ts +619 -0
- package/lib/recipes/locale.ts +600 -0
- package/lib/recipes/nl-postcode.ts +117 -0
- package/lib/recipes/no-fragment.ts +213 -0
- package/lib/recipes/no-street-led.ts +135 -0
- package/lib/recipes/no-street.ts +91 -0
- package/lib/recipes/po-box-cedex.ts +976 -0
- package/lib/recipes/po-box.ts +124 -0
- package/lib/recipes/reviewed-postcode-tail.ts +190 -0
- package/lib/recipes/scaffold.ts +468 -0
- package/lib/recipes/si-bare-village.ts +99 -0
- package/lib/recipes/street-affix.ts +805 -0
- package/lib/recipes/street-bare.ts +102 -0
- package/lib/recipes/street.ts +83 -0
- package/lib/recipes/sub-venue-sources.ts +878 -0
- package/lib/recipes/sub-venue.ts +998 -0
- package/lib/recipes/trailing-region.ts +178 -0
- package/lib/recipes/unit.ts +304 -0
- package/lib/runner.ts +312 -0
- package/lib/synthesizers/anchor-absorption.ts +288 -0
- package/lib/synthesizers/boundary-stress.ts +532 -0
- package/lib/synthesizers/german.ts +260 -0
- package/lib/synthesizers/house-venue.ts +308 -0
- package/lib/synthesizers/intersection.ts +233 -0
- package/lib/synthesizers/no-street.ts +327 -0
- package/lib/synthesizers/po-box.ts +323 -0
- package/lib/synthesizers/street.ts +269 -0
- package/lib/synthesizers/utils.ts +993 -0
- package/lib/test-kit/corpus-recipe.ts +86 -0
- package/lib/test-kit/index.ts +100 -0
- package/lib/tools/align-slice.ts +68 -0
- package/lib/tools/audit.ts +403 -0
- package/lib/tools/corpus-stats.ts +136 -0
- package/lib/tools/fetch/acra-sg.ts +174 -0
- package/lib/tools/fetch/ban.ts +289 -0
- package/lib/tools/fetch/download.ts +477 -0
- package/lib/tools/fetch/gcis-tw.ts +175 -0
- package/lib/tools/fetch/geonames-dump.ts +257 -0
- package/lib/tools/fetch/geonames-postal.ts +180 -0
- package/lib/tools/fetch/houjin-jp.ts +135 -0
- package/lib/tools/fetch/hrsa.ts +59 -0
- package/lib/tools/fetch/imls-pls.ts +149 -0
- package/lib/tools/fetch/index.ts +174 -0
- package/lib/tools/fetch/juso-kr.ts +165 -0
- package/lib/tools/fetch/localdata-kr.ts +194 -0
- package/lib/tools/fetch/nad.ts +341 -0
- package/lib/tools/fetch/nppes.ts +165 -0
- package/lib/tools/fetch/openaddresses.ts +347 -0
- package/lib/tools/fetch/ourairports.ts +170 -0
- package/lib/tools/fetch/ppd.ts +121 -0
- package/lib/tools/fetch/state-hi-schools.ts +161 -0
- package/lib/tools/fetch/state-sources.ts +146 -0
- package/lib/tools/fetch/tiger-full.ts +308 -0
- package/lib/tools/fetch/wikidata-subvenue.ts +384 -0
- package/lib/tools/golden-expand.ts +574 -0
- package/lib/tools/golden-promote.ts +337 -0
- package/lib/tools/golden-relabel-street.ts +686 -0
- package/lib/tools/index.ts +28 -0
- package/lib/tools/ingest-csv.ts +378 -0
- package/lib/tools/jsonl-to-parquet.ts +207 -0
- package/lib/tools/lint-slice-vocab.ts +388 -0
- package/lib/tools/lint-slice.ts +496 -0
- package/lib/tools/overlay-manifest.ts +148 -0
- package/lib/tools/overture-subvenue.ts +209 -0
- package/lib/tools/postcode-triples.ts +363 -0
- package/lib/tools/slice-kryptonite.ts +137 -0
- package/lib/tools/slice-translit.ts +218 -0
- package/lib/tools/sub-venue/harvest.ts +234 -0
- package/lib/tools/sub-venue/head-nouns.ts +245 -0
- package/lib/tools/sub-venue/surfaces.ts +95 -0
- package/lib/tools/sub-venue/table.ts +281 -0
- package/lib/tools/sub-venue/wikidata.ts +85 -0
- package/lib/tools/sub-venue-lexicon.ts +429 -0
- package/lib/tools/sub-venue-promotions.ts +330 -0
- package/lib/types.ts +247 -0
- package/lib/utils/align.ts +330 -0
- package/lib/utils/golden.ts +189 -0
- package/lib/utils/index.ts +14 -0
- package/lib/utils/parquet.ts +582 -0
- package/lib/utils/slice-stats.ts +95 -0
- package/lib/utils/split.ts +279 -0
- package/lib/utils/tokenize.ts +120 -0
- package/lib/utils/wof-json.ts +247 -0
- package/out/adapters/ban/adapter.d.ts.map +1 -0
- package/out/adapters/ban/adapter.js +149 -0
- package/out/adapters/ban/adapter.js.map +1 -0
- package/out/adapters/ban/street-decompose.d.ts.map +1 -0
- package/out/adapters/ban/street-decompose.js +51 -0
- package/out/adapters/ban/street-decompose.js.map +1 -0
- package/out/adapters/fcc-bdc/adapter.d.ts.map +1 -0
- package/out/adapters/fcc-bdc/adapter.js +144 -0
- package/out/adapters/fcc-bdc/adapter.js.map +1 -0
- package/out/adapters/geonames/adapter.d.ts.map +1 -0
- package/out/adapters/geonames/adapter.js +176 -0
- package/out/adapters/geonames/adapter.js.map +1 -0
- package/out/adapters/geonames-postal/adapter.d.ts +54 -0
- package/out/adapters/geonames-postal/adapter.d.ts.map +1 -0
- package/out/adapters/geonames-postal/adapter.js +107 -0
- package/out/adapters/geonames-postal/adapter.js.map +1 -0
- package/out/adapters/gnaf/adapter.d.ts +49 -0
- package/out/adapters/gnaf/adapter.d.ts.map +1 -0
- package/out/adapters/gnaf/adapter.js +124 -0
- package/out/adapters/gnaf/adapter.js.map +1 -0
- package/out/adapters/gnaf/assemble.d.ts +65 -0
- package/out/adapters/gnaf/assemble.d.ts.map +1 -0
- package/out/adapters/gnaf/assemble.js +154 -0
- package/out/adapters/gnaf/assemble.js.map +1 -0
- package/out/adapters/index.d.ts +51 -0
- package/out/adapters/index.d.ts.map +1 -0
- package/out/adapters/index.js +95 -0
- package/out/adapters/index.js.map +1 -0
- package/out/adapters/openaddresses/adapter.d.ts.map +1 -0
- package/out/adapters/openaddresses/adapter.js +186 -0
- package/out/adapters/openaddresses/adapter.js.map +1 -0
- package/out/adapters/overture/adapter.d.ts +48 -0
- package/out/adapters/overture/adapter.d.ts.map +1 -0
- package/out/adapters/overture/adapter.js +125 -0
- package/out/adapters/overture/adapter.js.map +1 -0
- package/out/adapters/state-hi-schools/adapter.d.ts +46 -0
- package/out/adapters/state-hi-schools/adapter.d.ts.map +1 -0
- package/out/adapters/state-hi-schools/adapter.js +145 -0
- package/out/adapters/state-hi-schools/adapter.js.map +1 -0
- package/out/adapters/state-hi-schools/workbook.d.ts +29 -0
- package/out/adapters/state-hi-schools/workbook.d.ts.map +1 -0
- package/out/adapters/state-hi-schools/workbook.js +20 -0
- package/out/adapters/state-hi-schools/workbook.js.map +1 -0
- package/out/adapters/state-ia-contractors/adapter.d.ts.map +1 -0
- package/out/adapters/state-ia-contractors/adapter.js.map +1 -0
- package/out/adapters/state-ny-notaries/adapter.d.ts.map +1 -0
- package/out/adapters/state-ny-notaries/adapter.js.map +1 -0
- package/out/adapters/state-tx-notaries/adapter.d.ts.map +1 -0
- package/out/adapters/state-tx-notaries/adapter.js.map +1 -0
- package/out/adapters/synth-po-box/adapter.d.ts.map +1 -0
- package/out/adapters/synth-po-box/adapter.js +140 -0
- package/out/adapters/synth-po-box/adapter.js.map +1 -0
- package/out/adapters/tiger/adapter.d.ts +55 -0
- package/out/adapters/tiger/adapter.d.ts.map +1 -0
- package/out/adapters/tiger/adapter.js +201 -0
- package/out/adapters/tiger/adapter.js.map +1 -0
- package/out/adapters/tiger/street-decompose.d.ts.map +1 -0
- package/out/adapters/tiger/street-decompose.js +68 -0
- package/out/adapters/tiger/street-decompose.js.map +1 -0
- package/out/adapters/usgov-hrsa-fqhc/adapter.d.ts +56 -0
- package/out/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-hrsa-fqhc/adapter.js +132 -0
- package/out/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -0
- package/out/adapters/usgov-imls-pls/adapter.d.ts +36 -0
- package/out/adapters/usgov-imls-pls/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-imls-pls/adapter.js +114 -0
- package/out/adapters/usgov-imls-pls/adapter.js.map +1 -0
- package/out/adapters/usgov-irs-bmf/adapter.d.ts +37 -0
- package/out/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-irs-bmf/adapter.js +120 -0
- package/out/adapters/usgov-irs-bmf/adapter.js.map +1 -0
- package/out/adapters/usgov-nad/adapter.d.ts +48 -0
- package/out/adapters/usgov-nad/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-nad/adapter.js +249 -0
- package/out/adapters/usgov-nad/adapter.js.map +1 -0
- package/out/adapters/usgov-nppes/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-nppes/adapter.js +118 -0
- package/out/adapters/usgov-nppes/adapter.js.map +1 -0
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -0
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.js +154 -0
- package/out/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -0
- package/out/adapters/utils/index.d.ts +191 -0
- package/out/adapters/utils/index.d.ts.map +1 -0
- package/out/adapters/utils/index.js +206 -0
- package/out/adapters/utils/index.js.map +1 -0
- package/out/adapters/wof-admin-jp/adapter.d.ts.map +1 -0
- package/out/adapters/wof-admin-jp/adapter.js +161 -0
- package/out/adapters/wof-admin-jp/adapter.js.map +1 -0
- package/out/adapters/wof-admin-json/adapter.d.ts +79 -0
- package/out/adapters/wof-admin-json/adapter.d.ts.map +1 -0
- package/out/adapters/wof-admin-json/adapter.js +197 -0
- package/out/adapters/wof-admin-json/adapter.js.map +1 -0
- package/out/adapters/wof-json-rows.d.ts +62 -0
- package/out/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/adapters/wof-json-rows.js +83 -0
- package/out/adapters/wof-json-rows.js.map +1 -0
- package/out/adapters/wof-postalcode-json/adapter.d.ts +65 -0
- package/out/adapters/wof-postalcode-json/adapter.d.ts.map +1 -0
- package/out/adapters/wof-postalcode-json/adapter.js +151 -0
- package/out/adapters/wof-postalcode-json/adapter.js.map +1 -0
- package/out/build.d.ts +132 -0
- package/out/build.d.ts.map +1 -0
- package/out/build.js +245 -0
- package/out/build.js.map +1 -0
- package/out/codex/us-fips-state.d.ts.map +1 -0
- package/out/codex/us-fips-state.js.map +1 -0
- package/out/env.d.ts +46 -0
- package/out/env.d.ts.map +1 -0
- package/out/env.js +46 -0
- package/out/env.js.map +1 -0
- package/out/index.d.ts +13 -0
- package/out/index.d.ts.map +1 -0
- package/out/index.js +13 -0
- package/out/index.js.map +1 -0
- package/out/recipes/anchor-absorption.d.ts +20 -0
- package/out/recipes/anchor-absorption.d.ts.map +1 -0
- package/out/recipes/anchor-absorption.js +82 -0
- package/out/recipes/anchor-absorption.js.map +1 -0
- package/out/recipes/bare-country.d.ts +25 -0
- package/out/recipes/bare-country.d.ts.map +1 -0
- package/out/recipes/bare-country.js +88 -0
- package/out/recipes/bare-country.js.map +1 -0
- package/out/recipes/boundary-stress.d.ts +20 -0
- package/out/recipes/boundary-stress.d.ts.map +1 -0
- package/out/recipes/boundary-stress.js +84 -0
- package/out/recipes/boundary-stress.js.map +1 -0
- package/out/recipes/cn-organizational-units.d.ts +36 -0
- package/out/recipes/cn-organizational-units.d.ts.map +1 -0
- package/out/recipes/cn-organizational-units.js +159 -0
- package/out/recipes/cn-organizational-units.js.map +1 -0
- package/out/recipes/country-balanced.d.ts +32 -0
- package/out/recipes/country-balanced.d.ts.map +1 -0
- package/out/recipes/country-balanced.js +394 -0
- package/out/recipes/country-balanced.js.map +1 -0
- package/out/recipes/cz-pcfirst-preposition.d.ts +22 -0
- package/out/recipes/cz-pcfirst-preposition.d.ts.map +1 -0
- package/out/recipes/cz-pcfirst-preposition.js +101 -0
- package/out/recipes/cz-pcfirst-preposition.js.map +1 -0
- package/out/recipes/fr-admin-split.d.ts +38 -0
- package/out/recipes/fr-admin-split.d.ts.map +1 -0
- package/out/recipes/fr-admin-split.js +192 -0
- package/out/recipes/fr-admin-split.js.map +1 -0
- package/out/recipes/fr-bare-street.d.ts +28 -0
- package/out/recipes/fr-bare-street.d.ts.map +1 -0
- package/out/recipes/fr-bare-street.js +144 -0
- package/out/recipes/fr-bare-street.js.map +1 -0
- package/out/recipes/fr-fragment.d.ts +77 -0
- package/out/recipes/fr-fragment.d.ts.map +1 -0
- package/out/recipes/fr-fragment.js +277 -0
- package/out/recipes/fr-fragment.js.map +1 -0
- package/out/recipes/fr-lieudit.d.ts +32 -0
- package/out/recipes/fr-lieudit.d.ts.map +1 -0
- package/out/recipes/fr-lieudit.js +227 -0
- package/out/recipes/fr-lieudit.js.map +1 -0
- package/out/recipes/fr-order.d.ts +38 -0
- package/out/recipes/fr-order.d.ts.map +1 -0
- package/out/recipes/fr-order.js +198 -0
- package/out/recipes/fr-order.js.map +1 -0
- package/out/recipes/german.d.ts +44 -0
- package/out/recipes/german.d.ts.map +1 -0
- package/out/recipes/german.js +289 -0
- package/out/recipes/german.js.map +1 -0
- package/out/recipes/house-venue.d.ts +18 -0
- package/out/recipes/house-venue.d.ts.map +1 -0
- package/out/recipes/house-venue.js +74 -0
- package/out/recipes/house-venue.js.map +1 -0
- package/out/recipes/index.d.ts +24 -0
- package/out/recipes/index.d.ts.map +1 -0
- package/out/recipes/index.js +91 -0
- package/out/recipes/index.js.map +1 -0
- package/out/recipes/intersection.d.ts +42 -0
- package/out/recipes/intersection.d.ts.map +1 -0
- package/out/recipes/intersection.js +453 -0
- package/out/recipes/intersection.js.map +1 -0
- package/out/recipes/locale.d.ts +136 -0
- package/out/recipes/locale.d.ts.map +1 -0
- package/out/recipes/locale.js +460 -0
- package/out/recipes/locale.js.map +1 -0
- package/out/recipes/nl-postcode.d.ts +26 -0
- package/out/recipes/nl-postcode.d.ts.map +1 -0
- package/out/recipes/nl-postcode.js +103 -0
- package/out/recipes/nl-postcode.js.map +1 -0
- package/out/recipes/no-fragment.d.ts +43 -0
- package/out/recipes/no-fragment.d.ts.map +1 -0
- package/out/recipes/no-fragment.js +183 -0
- package/out/recipes/no-fragment.js.map +1 -0
- package/out/recipes/no-street-led.d.ts +24 -0
- package/out/recipes/no-street-led.d.ts.map +1 -0
- package/out/recipes/no-street-led.js +114 -0
- package/out/recipes/no-street-led.js.map +1 -0
- package/out/recipes/no-street.d.ts +18 -0
- package/out/recipes/no-street.d.ts.map +1 -0
- package/out/recipes/no-street.js +72 -0
- package/out/recipes/no-street.js.map +1 -0
- package/out/recipes/po-box-cedex.d.ts +38 -0
- package/out/recipes/po-box-cedex.d.ts.map +1 -0
- package/out/recipes/po-box-cedex.js +768 -0
- package/out/recipes/po-box-cedex.js.map +1 -0
- package/out/recipes/po-box.d.ts +17 -0
- package/out/recipes/po-box.d.ts.map +1 -0
- package/out/recipes/po-box.js +101 -0
- package/out/recipes/po-box.js.map +1 -0
- package/out/recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/recipes/reviewed-postcode-tail.js +125 -0
- package/out/recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/recipes/scaffold.d.ts +323 -0
- package/out/recipes/scaffold.d.ts.map +1 -0
- package/out/recipes/scaffold.js +174 -0
- package/out/recipes/scaffold.js.map +1 -0
- package/out/recipes/si-bare-village.d.ts +32 -0
- package/out/recipes/si-bare-village.d.ts.map +1 -0
- package/out/recipes/si-bare-village.js +89 -0
- package/out/recipes/si-bare-village.js.map +1 -0
- package/out/recipes/street-affix.d.ts +89 -0
- package/out/recipes/street-affix.d.ts.map +1 -0
- package/out/recipes/street-affix.js +563 -0
- package/out/recipes/street-affix.js.map +1 -0
- package/out/recipes/street-bare.d.ts +25 -0
- package/out/recipes/street-bare.d.ts.map +1 -0
- package/out/recipes/street-bare.js +87 -0
- package/out/recipes/street-bare.js.map +1 -0
- package/out/recipes/street.d.ts +15 -0
- package/out/recipes/street.d.ts.map +1 -0
- package/out/recipes/street.js +70 -0
- package/out/recipes/street.js.map +1 -0
- package/out/recipes/sub-venue-sources.d.ts +229 -0
- package/out/recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/recipes/sub-venue-sources.js +654 -0
- package/out/recipes/sub-venue-sources.js.map +1 -0
- package/out/recipes/sub-venue.d.ts +269 -0
- package/out/recipes/sub-venue.d.ts.map +1 -0
- package/out/recipes/sub-venue.js +726 -0
- package/out/recipes/sub-venue.js.map +1 -0
- package/out/recipes/trailing-region.d.ts +74 -0
- package/out/recipes/trailing-region.d.ts.map +1 -0
- package/out/recipes/trailing-region.js +158 -0
- package/out/recipes/trailing-region.js.map +1 -0
- package/out/recipes/unit.d.ts +29 -0
- package/out/recipes/unit.d.ts.map +1 -0
- package/out/recipes/unit.js +231 -0
- package/out/recipes/unit.js.map +1 -0
- package/out/runner.d.ts +130 -0
- package/out/runner.d.ts.map +1 -0
- package/out/runner.js +185 -0
- package/out/runner.js.map +1 -0
- package/out/synthesizers/anchor-absorption.d.ts +63 -0
- package/out/synthesizers/anchor-absorption.d.ts.map +1 -0
- package/out/synthesizers/anchor-absorption.js +221 -0
- package/out/synthesizers/anchor-absorption.js.map +1 -0
- package/out/synthesizers/boundary-stress.d.ts +78 -0
- package/out/synthesizers/boundary-stress.d.ts.map +1 -0
- package/out/synthesizers/boundary-stress.js +455 -0
- package/out/synthesizers/boundary-stress.js.map +1 -0
- package/out/synthesizers/german.d.ts +115 -0
- package/out/synthesizers/german.d.ts.map +1 -0
- package/out/synthesizers/german.js +149 -0
- package/out/synthesizers/german.js.map +1 -0
- package/out/synthesizers/house-venue.d.ts +56 -0
- package/out/synthesizers/house-venue.d.ts.map +1 -0
- package/out/synthesizers/house-venue.js +247 -0
- package/out/synthesizers/house-venue.js.map +1 -0
- package/out/synthesizers/intersection.d.ts +52 -0
- package/out/synthesizers/intersection.d.ts.map +1 -0
- package/out/synthesizers/intersection.js +175 -0
- package/out/synthesizers/intersection.js.map +1 -0
- package/out/synthesizers/no-street.d.ts +72 -0
- package/out/synthesizers/no-street.d.ts.map +1 -0
- package/out/synthesizers/no-street.js +262 -0
- package/out/synthesizers/no-street.js.map +1 -0
- package/out/synthesizers/po-box.d.ts +108 -0
- package/out/synthesizers/po-box.d.ts.map +1 -0
- package/out/synthesizers/po-box.js +250 -0
- package/out/synthesizers/po-box.js.map +1 -0
- package/out/synthesizers/street.d.ts +54 -0
- package/out/synthesizers/street.d.ts.map +1 -0
- package/out/synthesizers/street.js +218 -0
- package/out/synthesizers/street.js.map +1 -0
- package/out/synthesizers/utils.d.ts +241 -0
- package/out/synthesizers/utils.d.ts.map +1 -0
- package/out/synthesizers/utils.js +776 -0
- package/out/synthesizers/utils.js.map +1 -0
- package/out/test-kit/corpus-recipe.d.ts +59 -0
- package/out/test-kit/corpus-recipe.d.ts.map +1 -0
- package/out/test-kit/corpus-recipe.js +39 -0
- package/out/test-kit/corpus-recipe.js.map +1 -0
- package/out/test-kit/index.d.ts +48 -0
- package/out/test-kit/index.d.ts.map +1 -0
- package/out/test-kit/index.js +78 -0
- package/out/test-kit/index.js.map +1 -0
- package/out/tools/align-slice.d.ts +7 -0
- package/out/tools/align-slice.d.ts.map +1 -0
- package/out/tools/align-slice.js +55 -0
- package/out/tools/align-slice.js.map +1 -0
- package/out/tools/audit.d.ts +29 -0
- package/out/tools/audit.d.ts.map +1 -0
- package/out/tools/audit.js +282 -0
- package/out/tools/audit.js.map +1 -0
- package/out/tools/corpus-stats.d.ts +44 -0
- package/out/tools/corpus-stats.d.ts.map +1 -0
- package/out/tools/corpus-stats.js +102 -0
- package/out/tools/corpus-stats.js.map +1 -0
- package/out/tools/fetch/acra-sg.d.ts +23 -0
- package/out/tools/fetch/acra-sg.d.ts.map +1 -0
- package/out/tools/fetch/acra-sg.js +126 -0
- package/out/tools/fetch/acra-sg.js.map +1 -0
- package/out/tools/fetch/ban.d.ts +31 -0
- package/out/tools/fetch/ban.d.ts.map +1 -0
- package/out/tools/fetch/ban.js +239 -0
- package/out/tools/fetch/ban.js.map +1 -0
- package/out/tools/fetch/download.d.ts +182 -0
- package/out/tools/fetch/download.d.ts.map +1 -0
- package/out/tools/fetch/download.js +318 -0
- package/out/tools/fetch/download.js.map +1 -0
- package/out/tools/fetch/gcis-tw.d.ts +23 -0
- package/out/tools/fetch/gcis-tw.d.ts.map +1 -0
- package/out/tools/fetch/gcis-tw.js +135 -0
- package/out/tools/fetch/gcis-tw.js.map +1 -0
- package/out/tools/fetch/geonames-dump.d.ts +91 -0
- package/out/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/tools/fetch/geonames-dump.js +171 -0
- package/out/tools/fetch/geonames-dump.js.map +1 -0
- package/out/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/tools/fetch/geonames-postal.js +126 -0
- package/out/tools/fetch/geonames-postal.js.map +1 -0
- package/out/tools/fetch/houjin-jp.d.ts +24 -0
- package/out/tools/fetch/houjin-jp.d.ts.map +1 -0
- package/out/tools/fetch/houjin-jp.js +102 -0
- package/out/tools/fetch/houjin-jp.js.map +1 -0
- package/out/tools/fetch/hrsa.d.ts +18 -0
- package/out/tools/fetch/hrsa.d.ts.map +1 -0
- package/out/tools/fetch/hrsa.js +46 -0
- package/out/tools/fetch/hrsa.js.map +1 -0
- package/out/tools/fetch/imls-pls.d.ts +25 -0
- package/out/tools/fetch/imls-pls.d.ts.map +1 -0
- package/out/tools/fetch/imls-pls.js +108 -0
- package/out/tools/fetch/imls-pls.js.map +1 -0
- package/out/tools/fetch/index.d.ts +171 -0
- package/out/tools/fetch/index.d.ts.map +1 -0
- package/out/tools/fetch/index.js +170 -0
- package/out/tools/fetch/index.js.map +1 -0
- package/out/tools/fetch/juso-kr.d.ts +30 -0
- package/out/tools/fetch/juso-kr.d.ts.map +1 -0
- package/out/tools/fetch/juso-kr.js +120 -0
- package/out/tools/fetch/juso-kr.js.map +1 -0
- package/out/tools/fetch/localdata-kr.d.ts +30 -0
- package/out/tools/fetch/localdata-kr.d.ts.map +1 -0
- package/out/tools/fetch/localdata-kr.js +139 -0
- package/out/tools/fetch/localdata-kr.js.map +1 -0
- package/out/tools/fetch/nad.d.ts +70 -0
- package/out/tools/fetch/nad.d.ts.map +1 -0
- package/out/tools/fetch/nad.js +234 -0
- package/out/tools/fetch/nad.js.map +1 -0
- package/out/tools/fetch/nppes.d.ts +26 -0
- package/out/tools/fetch/nppes.d.ts.map +1 -0
- package/out/tools/fetch/nppes.js +126 -0
- package/out/tools/fetch/nppes.js.map +1 -0
- package/out/tools/fetch/openaddresses.d.ts +9 -0
- package/out/tools/fetch/openaddresses.d.ts.map +1 -0
- package/out/tools/fetch/openaddresses.js +279 -0
- package/out/tools/fetch/openaddresses.js.map +1 -0
- package/out/tools/fetch/ourairports.d.ts +45 -0
- package/out/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/tools/fetch/ourairports.js +128 -0
- package/out/tools/fetch/ourairports.js.map +1 -0
- package/out/tools/fetch/ppd.d.ts.map +1 -0
- package/out/tools/fetch/ppd.js +85 -0
- package/out/tools/fetch/ppd.js.map +1 -0
- package/out/tools/fetch/state-hi-schools.d.ts +26 -0
- package/out/tools/fetch/state-hi-schools.d.ts.map +1 -0
- package/out/tools/fetch/state-hi-schools.js +112 -0
- package/out/tools/fetch/state-hi-schools.js.map +1 -0
- package/out/tools/fetch/state-sources.d.ts +26 -0
- package/out/tools/fetch/state-sources.d.ts.map +1 -0
- package/out/tools/fetch/state-sources.js +111 -0
- package/out/tools/fetch/state-sources.js.map +1 -0
- package/out/tools/fetch/tiger-full.d.ts +22 -0
- package/out/tools/fetch/tiger-full.d.ts.map +1 -0
- package/out/tools/fetch/tiger-full.js +217 -0
- package/out/tools/fetch/tiger-full.js.map +1 -0
- package/out/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/tools/fetch/wikidata-subvenue.js +274 -0
- package/out/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/tools/gb-title-case.d.ts.map +1 -0
- package/out/tools/gb-title-case.js.map +1 -0
- package/out/tools/golden-expand.d.ts +90 -0
- package/out/tools/golden-expand.d.ts.map +1 -0
- package/out/tools/golden-expand.js +406 -0
- package/out/tools/golden-expand.js.map +1 -0
- package/out/tools/golden-promote.d.ts.map +1 -0
- package/out/tools/golden-promote.js +240 -0
- package/out/tools/golden-promote.js.map +1 -0
- package/out/tools/golden-relabel-street.d.ts +196 -0
- package/out/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/tools/golden-relabel-street.js +439 -0
- package/out/tools/golden-relabel-street.js.map +1 -0
- package/out/tools/index.d.ts +28 -0
- package/out/tools/index.d.ts.map +1 -0
- package/out/tools/index.js +28 -0
- package/out/tools/index.js.map +1 -0
- package/out/tools/ingest-csv.d.ts.map +1 -0
- package/out/tools/ingest-csv.js +265 -0
- package/out/tools/ingest-csv.js.map +1 -0
- package/out/tools/jsonl-to-parquet.d.ts +61 -0
- package/out/tools/jsonl-to-parquet.d.ts.map +1 -0
- package/out/tools/jsonl-to-parquet.js +150 -0
- package/out/tools/jsonl-to-parquet.js.map +1 -0
- package/out/tools/lint-slice-vocab.d.ts +90 -0
- package/out/tools/lint-slice-vocab.d.ts.map +1 -0
- package/out/tools/lint-slice-vocab.js +267 -0
- package/out/tools/lint-slice-vocab.js.map +1 -0
- package/out/tools/lint-slice.d.ts +93 -0
- package/out/tools/lint-slice.d.ts.map +1 -0
- package/out/tools/lint-slice.js +322 -0
- package/out/tools/lint-slice.js.map +1 -0
- package/out/tools/overlay-manifest.d.ts +39 -0
- package/out/tools/overlay-manifest.d.ts.map +1 -0
- package/out/tools/overlay-manifest.js +90 -0
- package/out/tools/overlay-manifest.js.map +1 -0
- package/out/tools/overture-subvenue.d.ts +112 -0
- package/out/tools/overture-subvenue.d.ts.map +1 -0
- package/out/tools/overture-subvenue.js +139 -0
- package/out/tools/overture-subvenue.js.map +1 -0
- package/out/tools/postcode-triples.d.ts +174 -0
- package/out/tools/postcode-triples.d.ts.map +1 -0
- package/out/tools/postcode-triples.js +292 -0
- package/out/tools/postcode-triples.js.map +1 -0
- package/out/tools/slice-kryptonite.d.ts +32 -0
- package/out/tools/slice-kryptonite.d.ts.map +1 -0
- package/out/tools/slice-kryptonite.js +99 -0
- package/out/tools/slice-kryptonite.js.map +1 -0
- package/out/tools/slice-translit.d.ts +51 -0
- package/out/tools/slice-translit.d.ts.map +1 -0
- package/out/tools/slice-translit.js +166 -0
- package/out/tools/slice-translit.js.map +1 -0
- package/out/tools/sub-venue/harvest.d.ts +100 -0
- package/out/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/tools/sub-venue/harvest.js +168 -0
- package/out/tools/sub-venue/harvest.js.map +1 -0
- package/out/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/tools/sub-venue/head-nouns.js +213 -0
- package/out/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/tools/sub-venue/surfaces.js +77 -0
- package/out/tools/sub-venue/surfaces.js.map +1 -0
- package/out/tools/sub-venue/table.d.ts +229 -0
- package/out/tools/sub-venue/table.d.ts.map +1 -0
- package/out/tools/sub-venue/table.js +108 -0
- package/out/tools/sub-venue/table.js.map +1 -0
- package/out/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/tools/sub-venue/wikidata.js +62 -0
- package/out/tools/sub-venue/wikidata.js.map +1 -0
- package/out/tools/sub-venue-lexicon.d.ts +156 -0
- package/out/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/tools/sub-venue-lexicon.js +283 -0
- package/out/tools/sub-venue-lexicon.js.map +1 -0
- package/out/tools/sub-venue-promotions.d.ts +94 -0
- package/out/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/tools/sub-venue-promotions.js +266 -0
- package/out/tools/sub-venue-promotions.js.map +1 -0
- package/out/types.d.ts +221 -0
- package/out/types.d.ts.map +1 -0
- package/out/types.js.map +1 -0
- package/out/utils/align.d.ts +93 -0
- package/out/utils/align.d.ts.map +1 -0
- package/out/utils/align.js +234 -0
- package/out/utils/align.js.map +1 -0
- package/out/utils/golden.d.ts +85 -0
- package/out/utils/golden.d.ts.map +1 -0
- package/out/utils/golden.js +127 -0
- package/out/utils/golden.js.map +1 -0
- package/out/utils/index.d.ts +14 -0
- package/out/utils/index.d.ts.map +1 -0
- package/out/utils/index.js +14 -0
- package/out/utils/index.js.map +1 -0
- package/out/utils/license.d.ts.map +1 -0
- package/out/utils/license.js.map +1 -0
- package/out/utils/parquet.d.ts +178 -0
- package/out/utils/parquet.d.ts.map +1 -0
- package/out/utils/parquet.js +371 -0
- package/out/utils/parquet.js.map +1 -0
- package/out/utils/slice-stats.d.ts +52 -0
- package/out/utils/slice-stats.d.ts.map +1 -0
- package/out/utils/slice-stats.js +58 -0
- package/out/utils/slice-stats.js.map +1 -0
- package/out/utils/split.d.ts +128 -0
- package/out/utils/split.d.ts.map +1 -0
- package/out/utils/split.js +204 -0
- package/out/utils/split.js.map +1 -0
- package/out/utils/tokenize.d.ts +64 -0
- package/out/utils/tokenize.d.ts.map +1 -0
- package/out/utils/tokenize.js +82 -0
- package/out/utils/tokenize.js.map +1 -0
- package/out/utils/wof-json.d.ts +113 -0
- package/out/utils/wof-json.d.ts.map +1 -0
- package/out/utils/wof-json.js +173 -0
- package/out/utils/wof-json.js.map +1 -0
- package/package.json +259 -194
- package/out/src/adapters/ban/adapter.d.ts.map +0 -1
- package/out/src/adapters/ban/adapter.js +0 -149
- package/out/src/adapters/ban/adapter.js.map +0 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +0 -1
- package/out/src/adapters/ban/street-decompose.js +0 -75
- package/out/src/adapters/ban/street-decompose.js.map +0 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +0 -1
- package/out/src/adapters/fcc-bdc/adapter.js +0 -144
- package/out/src/adapters/fcc-bdc/adapter.js.map +0 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +0 -1
- package/out/src/adapters/geonames/adapter.js +0 -176
- package/out/src/adapters/geonames/adapter.js.map +0 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +0 -41
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +0 -1
- package/out/src/adapters/geonames-postal/adapter.js +0 -104
- package/out/src/adapters/geonames-postal/adapter.js.map +0 -1
- package/out/src/adapters/gnaf/adapter.d.ts +0 -49
- package/out/src/adapters/gnaf/adapter.d.ts.map +0 -1
- package/out/src/adapters/gnaf/adapter.js +0 -124
- package/out/src/adapters/gnaf/adapter.js.map +0 -1
- package/out/src/adapters/gnaf/assemble.d.ts +0 -65
- package/out/src/adapters/gnaf/assemble.d.ts.map +0 -1
- package/out/src/adapters/gnaf/assemble.js +0 -154
- package/out/src/adapters/gnaf/assemble.js.map +0 -1
- package/out/src/adapters/index.d.ts +0 -51
- package/out/src/adapters/index.d.ts.map +0 -1
- package/out/src/adapters/index.js +0 -95
- package/out/src/adapters/index.js.map +0 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +0 -1
- package/out/src/adapters/openaddresses/adapter.js +0 -186
- package/out/src/adapters/openaddresses/adapter.js.map +0 -1
- package/out/src/adapters/overture/adapter.d.ts +0 -48
- package/out/src/adapters/overture/adapter.d.ts.map +0 -1
- package/out/src/adapters/overture/adapter.js +0 -125
- package/out/src/adapters/overture/adapter.js.map +0 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +0 -47
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +0 -1
- package/out/src/adapters/state-hi-schools/adapter.js +0 -133
- package/out/src/adapters/state-hi-schools/adapter.js.map +0 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +0 -1
- package/out/src/adapters/state-ia-contractors/adapter.js.map +0 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +0 -1
- package/out/src/adapters/state-ny-notaries/adapter.js.map +0 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +0 -1
- package/out/src/adapters/state-tx-notaries/adapter.js.map +0 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +0 -1
- package/out/src/adapters/synth-po-box/adapter.js +0 -140
- package/out/src/adapters/synth-po-box/adapter.js.map +0 -1
- package/out/src/adapters/tiger/adapter.d.ts +0 -55
- package/out/src/adapters/tiger/adapter.d.ts.map +0 -1
- package/out/src/adapters/tiger/adapter.js +0 -201
- package/out/src/adapters/tiger/adapter.js.map +0 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +0 -1
- package/out/src/adapters/tiger/street-decompose.js +0 -90
- package/out/src/adapters/tiger/street-decompose.js.map +0 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +0 -47
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +0 -139
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +0 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +0 -36
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +0 -114
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +0 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +0 -37
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +0 -124
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +0 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +0 -48
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-nad/adapter.js +0 -252
- package/out/src/adapters/usgov-nad/adapter.js.map +0 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-nppes/adapter.js +0 -116
- package/out/src/adapters/usgov-nppes/adapter.js.map +0 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +0 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +0 -156
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +0 -1
- package/out/src/adapters/utils/index.d.ts +0 -150
- package/out/src/adapters/utils/index.d.ts.map +0 -1
- package/out/src/adapters/utils/index.js +0 -159
- package/out/src/adapters/utils/index.js.map +0 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +0 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +0 -164
- package/out/src/adapters/wof-admin-jp/adapter.js.map +0 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +0 -82
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +0 -1
- package/out/src/adapters/wof-admin-json/adapter.js +0 -224
- package/out/src/adapters/wof-admin-json/adapter.js.map +0 -1
- package/out/src/adapters/wof-json-rows.d.ts +0 -32
- package/out/src/adapters/wof-json-rows.d.ts.map +0 -1
- package/out/src/adapters/wof-json-rows.js +0 -45
- package/out/src/adapters/wof-json-rows.js.map +0 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +0 -65
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +0 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +0 -160
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +0 -1
- package/out/src/build.d.ts +0 -132
- package/out/src/build.d.ts.map +0 -1
- package/out/src/build.js +0 -251
- package/out/src/build.js.map +0 -1
- package/out/src/codex/us-fips-state.d.ts.map +0 -1
- package/out/src/codex/us-fips-state.js.map +0 -1
- package/out/src/index.d.ts +0 -14
- package/out/src/index.d.ts.map +0 -1
- package/out/src/index.js +0 -14
- package/out/src/index.js.map +0 -1
- package/out/src/name-prone-us-suffixes.d.ts +0 -12
- package/out/src/name-prone-us-suffixes.d.ts.map +0 -1
- package/out/src/name-prone-us-suffixes.js +0 -12
- package/out/src/name-prone-us-suffixes.js.map +0 -1
- package/out/src/parquet-wrapper/index.d.ts +0 -9
- package/out/src/parquet-wrapper/index.d.ts.map +0 -1
- package/out/src/parquet-wrapper/index.js +0 -9
- package/out/src/parquet-wrapper/index.js.map +0 -1
- package/out/src/parquet-wrapper/reader.d.ts +0 -34
- package/out/src/parquet-wrapper/reader.d.ts.map +0 -1
- package/out/src/parquet-wrapper/reader.js +0 -69
- package/out/src/parquet-wrapper/reader.js.map +0 -1
- package/out/src/parquet-wrapper/schema.d.ts +0 -49
- package/out/src/parquet-wrapper/schema.d.ts.map +0 -1
- package/out/src/parquet-wrapper/schema.js +0 -55
- package/out/src/parquet-wrapper/schema.js.map +0 -1
- package/out/src/parquet-wrapper/writer.d.ts +0 -46
- package/out/src/parquet-wrapper/writer.d.ts.map +0 -1
- package/out/src/parquet-wrapper/writer.js +0 -76
- package/out/src/parquet-wrapper/writer.js.map +0 -1
- package/out/src/runner.d.ts +0 -126
- package/out/src/runner.d.ts.map +0 -1
- package/out/src/runner.js +0 -189
- package/out/src/runner.js.map +0 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +0 -20
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +0 -1
- package/out/src/shard-recipes/anchor-absorption.js +0 -82
- package/out/src/shard-recipes/anchor-absorption.js.map +0 -1
- package/out/src/shard-recipes/bare-country.d.ts +0 -25
- package/out/src/shard-recipes/bare-country.d.ts.map +0 -1
- package/out/src/shard-recipes/bare-country.js +0 -87
- package/out/src/shard-recipes/bare-country.js.map +0 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +0 -20
- package/out/src/shard-recipes/boundary-stress.d.ts.map +0 -1
- package/out/src/shard-recipes/boundary-stress.js +0 -83
- package/out/src/shard-recipes/boundary-stress.js.map +0 -1
- package/out/src/shard-recipes/country-balanced.d.ts +0 -32
- package/out/src/shard-recipes/country-balanced.d.ts.map +0 -1
- package/out/src/shard-recipes/country-balanced.js +0 -406
- package/out/src/shard-recipes/country-balanced.js.map +0 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +0 -22
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +0 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +0 -100
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +0 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +0 -38
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +0 -1
- package/out/src/shard-recipes/fr-admin-split.js +0 -188
- package/out/src/shard-recipes/fr-admin-split.js.map +0 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +0 -28
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +0 -1
- package/out/src/shard-recipes/fr-bare-street.js +0 -143
- package/out/src/shard-recipes/fr-bare-street.js.map +0 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +0 -77
- package/out/src/shard-recipes/fr-fragment.d.ts.map +0 -1
- package/out/src/shard-recipes/fr-fragment.js +0 -276
- package/out/src/shard-recipes/fr-fragment.js.map +0 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +0 -32
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +0 -1
- package/out/src/shard-recipes/fr-lieudit.js +0 -226
- package/out/src/shard-recipes/fr-lieudit.js.map +0 -1
- package/out/src/shard-recipes/fr-order.d.ts +0 -38
- package/out/src/shard-recipes/fr-order.d.ts.map +0 -1
- package/out/src/shard-recipes/fr-order.js +0 -208
- package/out/src/shard-recipes/fr-order.js.map +0 -1
- package/out/src/shard-recipes/german.d.ts +0 -23
- package/out/src/shard-recipes/german.d.ts.map +0 -1
- package/out/src/shard-recipes/german.js +0 -132
- package/out/src/shard-recipes/german.js.map +0 -1
- package/out/src/shard-recipes/house-venue.d.ts +0 -18
- package/out/src/shard-recipes/house-venue.d.ts.map +0 -1
- package/out/src/shard-recipes/house-venue.js +0 -74
- package/out/src/shard-recipes/house-venue.js.map +0 -1
- package/out/src/shard-recipes/index.d.ts +0 -24
- package/out/src/shard-recipes/index.d.ts.map +0 -1
- package/out/src/shard-recipes/index.js +0 -89
- package/out/src/shard-recipes/index.js.map +0 -1
- package/out/src/shard-recipes/intersection.d.ts +0 -42
- package/out/src/shard-recipes/intersection.d.ts.map +0 -1
- package/out/src/shard-recipes/intersection.js +0 -460
- package/out/src/shard-recipes/intersection.js.map +0 -1
- package/out/src/shard-recipes/locale.d.ts +0 -135
- package/out/src/shard-recipes/locale.d.ts.map +0 -1
- package/out/src/shard-recipes/locale.js +0 -459
- package/out/src/shard-recipes/locale.js.map +0 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +0 -26
- package/out/src/shard-recipes/nl-postcode.d.ts.map +0 -1
- package/out/src/shard-recipes/nl-postcode.js +0 -102
- package/out/src/shard-recipes/nl-postcode.js.map +0 -1
- package/out/src/shard-recipes/no-fragment.d.ts +0 -43
- package/out/src/shard-recipes/no-fragment.d.ts.map +0 -1
- package/out/src/shard-recipes/no-fragment.js +0 -188
- package/out/src/shard-recipes/no-fragment.js.map +0 -1
- package/out/src/shard-recipes/no-street-led.d.ts +0 -24
- package/out/src/shard-recipes/no-street-led.d.ts.map +0 -1
- package/out/src/shard-recipes/no-street-led.js +0 -121
- package/out/src/shard-recipes/no-street-led.js.map +0 -1
- package/out/src/shard-recipes/no-street.d.ts +0 -18
- package/out/src/shard-recipes/no-street.d.ts.map +0 -1
- package/out/src/shard-recipes/no-street.js +0 -72
- package/out/src/shard-recipes/no-street.js.map +0 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +0 -38
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +0 -1
- package/out/src/shard-recipes/po-box-cedex.js +0 -726
- package/out/src/shard-recipes/po-box-cedex.js.map +0 -1
- package/out/src/shard-recipes/po-box.d.ts +0 -17
- package/out/src/shard-recipes/po-box.d.ts.map +0 -1
- package/out/src/shard-recipes/po-box.js +0 -101
- package/out/src/shard-recipes/po-box.js.map +0 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +0 -47
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +0 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.js +0 -125
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +0 -1
- package/out/src/shard-recipes/scaffold.d.ts +0 -246
- package/out/src/shard-recipes/scaffold.d.ts.map +0 -1
- package/out/src/shard-recipes/scaffold.js +0 -137
- package/out/src/shard-recipes/scaffold.js.map +0 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +0 -32
- package/out/src/shard-recipes/si-bare-village.d.ts.map +0 -1
- package/out/src/shard-recipes/si-bare-village.js +0 -88
- package/out/src/shard-recipes/si-bare-village.js.map +0 -1
- package/out/src/shard-recipes/street-affix.d.ts +0 -89
- package/out/src/shard-recipes/street-affix.d.ts.map +0 -1
- package/out/src/shard-recipes/street-affix.js +0 -578
- package/out/src/shard-recipes/street-affix.js.map +0 -1
- package/out/src/shard-recipes/street-bare.d.ts +0 -25
- package/out/src/shard-recipes/street-bare.d.ts.map +0 -1
- package/out/src/shard-recipes/street-bare.js +0 -86
- package/out/src/shard-recipes/street-bare.js.map +0 -1
- package/out/src/shard-recipes/street.d.ts +0 -15
- package/out/src/shard-recipes/street.d.ts.map +0 -1
- package/out/src/shard-recipes/street.js +0 -69
- package/out/src/shard-recipes/street.js.map +0 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +0 -228
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +0 -1
- package/out/src/shard-recipes/sub-venue-sources.js +0 -664
- package/out/src/shard-recipes/sub-venue-sources.js.map +0 -1
- package/out/src/shard-recipes/sub-venue.d.ts +0 -269
- package/out/src/shard-recipes/sub-venue.d.ts.map +0 -1
- package/out/src/shard-recipes/sub-venue.js +0 -710
- package/out/src/shard-recipes/sub-venue.js.map +0 -1
- package/out/src/shard-recipes/trailing-region.d.ts +0 -74
- package/out/src/shard-recipes/trailing-region.d.ts.map +0 -1
- package/out/src/shard-recipes/trailing-region.js +0 -157
- package/out/src/shard-recipes/trailing-region.js.map +0 -1
- package/out/src/shard-recipes/unit.d.ts +0 -29
- package/out/src/shard-recipes/unit.d.ts.map +0 -1
- package/out/src/shard-recipes/unit.js +0 -243
- package/out/src/shard-recipes/unit.js.map +0 -1
- package/out/src/synthesizers/anchor-absorption.d.ts +0 -63
- package/out/src/synthesizers/anchor-absorption.d.ts.map +0 -1
- package/out/src/synthesizers/anchor-absorption.js +0 -223
- package/out/src/synthesizers/anchor-absorption.js.map +0 -1
- package/out/src/synthesizers/boundary-stress.d.ts +0 -78
- package/out/src/synthesizers/boundary-stress.d.ts.map +0 -1
- package/out/src/synthesizers/boundary-stress.js +0 -455
- package/out/src/synthesizers/boundary-stress.js.map +0 -1
- package/out/src/synthesizers/german.d.ts +0 -107
- package/out/src/synthesizers/german.d.ts.map +0 -1
- package/out/src/synthesizers/german.js +0 -148
- package/out/src/synthesizers/german.js.map +0 -1
- package/out/src/synthesizers/house-venue.d.ts +0 -56
- package/out/src/synthesizers/house-venue.d.ts.map +0 -1
- package/out/src/synthesizers/house-venue.js +0 -267
- package/out/src/synthesizers/house-venue.js.map +0 -1
- package/out/src/synthesizers/intersection.d.ts +0 -52
- package/out/src/synthesizers/intersection.d.ts.map +0 -1
- package/out/src/synthesizers/intersection.js +0 -175
- package/out/src/synthesizers/intersection.js.map +0 -1
- package/out/src/synthesizers/no-street.d.ts +0 -72
- package/out/src/synthesizers/no-street.d.ts.map +0 -1
- package/out/src/synthesizers/no-street.js +0 -282
- package/out/src/synthesizers/no-street.js.map +0 -1
- package/out/src/synthesizers/po-box.d.ts +0 -104
- package/out/src/synthesizers/po-box.d.ts.map +0 -1
- package/out/src/synthesizers/po-box.js +0 -266
- package/out/src/synthesizers/po-box.js.map +0 -1
- package/out/src/synthesizers/street.d.ts +0 -54
- package/out/src/synthesizers/street.d.ts.map +0 -1
- package/out/src/synthesizers/street.js +0 -221
- package/out/src/synthesizers/street.js.map +0 -1
- package/out/src/synthesizers/utils.d.ts +0 -203
- package/out/src/synthesizers/utils.d.ts.map +0 -1
- package/out/src/synthesizers/utils.js +0 -708
- package/out/src/synthesizers/utils.js.map +0 -1
- package/out/src/tools/align-shard.d.ts +0 -8
- package/out/src/tools/align-shard.d.ts.map +0 -1
- package/out/src/tools/align-shard.js +0 -56
- package/out/src/tools/align-shard.js.map +0 -1
- package/out/src/tools/audit.d.ts +0 -28
- package/out/src/tools/audit.d.ts.map +0 -1
- package/out/src/tools/audit.js +0 -285
- package/out/src/tools/audit.js.map +0 -1
- package/out/src/tools/corpus-stats.d.ts +0 -44
- package/out/src/tools/corpus-stats.d.ts.map +0 -1
- package/out/src/tools/corpus-stats.js +0 -138
- package/out/src/tools/corpus-stats.js.map +0 -1
- package/out/src/tools/fetch/ban.d.ts +0 -31
- package/out/src/tools/fetch/ban.d.ts.map +0 -1
- package/out/src/tools/fetch/ban.js +0 -239
- package/out/src/tools/fetch/ban.js.map +0 -1
- package/out/src/tools/fetch/download.d.ts +0 -78
- package/out/src/tools/fetch/download.d.ts.map +0 -1
- package/out/src/tools/fetch/download.js +0 -116
- package/out/src/tools/fetch/download.js.map +0 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +0 -87
- package/out/src/tools/fetch/geonames-dump.d.ts.map +0 -1
- package/out/src/tools/fetch/geonames-dump.js +0 -178
- package/out/src/tools/fetch/geonames-dump.js.map +0 -1
- package/out/src/tools/fetch/geonames-postal.d.ts +0 -76
- package/out/src/tools/fetch/geonames-postal.d.ts.map +0 -1
- package/out/src/tools/fetch/geonames-postal.js +0 -125
- package/out/src/tools/fetch/geonames-postal.js.map +0 -1
- package/out/src/tools/fetch/hrsa.d.ts +0 -18
- package/out/src/tools/fetch/hrsa.d.ts.map +0 -1
- package/out/src/tools/fetch/hrsa.js +0 -46
- package/out/src/tools/fetch/hrsa.js.map +0 -1
- package/out/src/tools/fetch/imls-pls.d.ts +0 -25
- package/out/src/tools/fetch/imls-pls.d.ts.map +0 -1
- package/out/src/tools/fetch/imls-pls.js +0 -108
- package/out/src/tools/fetch/imls-pls.js.map +0 -1
- package/out/src/tools/fetch/index.d.ts +0 -144
- package/out/src/tools/fetch/index.d.ts.map +0 -1
- package/out/src/tools/fetch/index.js +0 -143
- package/out/src/tools/fetch/index.js.map +0 -1
- package/out/src/tools/fetch/nad.d.ts +0 -70
- package/out/src/tools/fetch/nad.d.ts.map +0 -1
- package/out/src/tools/fetch/nad.js +0 -234
- package/out/src/tools/fetch/nad.js.map +0 -1
- package/out/src/tools/fetch/nppes.d.ts +0 -26
- package/out/src/tools/fetch/nppes.d.ts.map +0 -1
- package/out/src/tools/fetch/nppes.js +0 -126
- package/out/src/tools/fetch/nppes.js.map +0 -1
- package/out/src/tools/fetch/openaddresses.d.ts +0 -61
- package/out/src/tools/fetch/openaddresses.d.ts.map +0 -1
- package/out/src/tools/fetch/openaddresses.js +0 -317
- package/out/src/tools/fetch/openaddresses.js.map +0 -1
- package/out/src/tools/fetch/ourairports.d.ts +0 -45
- package/out/src/tools/fetch/ourairports.d.ts.map +0 -1
- package/out/src/tools/fetch/ourairports.js +0 -128
- package/out/src/tools/fetch/ourairports.js.map +0 -1
- package/out/src/tools/fetch/ppd.d.ts.map +0 -1
- package/out/src/tools/fetch/ppd.js +0 -100
- package/out/src/tools/fetch/ppd.js.map +0 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts +0 -25
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +0 -1
- package/out/src/tools/fetch/state-hi-schools.js +0 -162
- package/out/src/tools/fetch/state-hi-schools.js.map +0 -1
- package/out/src/tools/fetch/state-sources.d.ts +0 -26
- package/out/src/tools/fetch/state-sources.d.ts.map +0 -1
- package/out/src/tools/fetch/state-sources.js +0 -111
- package/out/src/tools/fetch/state-sources.js.map +0 -1
- package/out/src/tools/fetch/tiger-full.d.ts +0 -42
- package/out/src/tools/fetch/tiger-full.d.ts.map +0 -1
- package/out/src/tools/fetch/tiger-full.js +0 -248
- package/out/src/tools/fetch/tiger-full.js.map +0 -1
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +0 -171
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +0 -1
- package/out/src/tools/fetch/wikidata-subvenue.js +0 -275
- package/out/src/tools/fetch/wikidata-subvenue.js.map +0 -1
- package/out/src/tools/gb-title-case.d.ts.map +0 -1
- package/out/src/tools/gb-title-case.js.map +0 -1
- package/out/src/tools/golden-expand.d.ts +0 -90
- package/out/src/tools/golden-expand.d.ts.map +0 -1
- package/out/src/tools/golden-expand.js +0 -413
- package/out/src/tools/golden-expand.js.map +0 -1
- package/out/src/tools/golden-promote.d.ts.map +0 -1
- package/out/src/tools/golden-promote.js +0 -244
- package/out/src/tools/golden-promote.js.map +0 -1
- package/out/src/tools/golden-relabel-street.d.ts +0 -196
- package/out/src/tools/golden-relabel-street.d.ts.map +0 -1
- package/out/src/tools/golden-relabel-street.js +0 -440
- package/out/src/tools/golden-relabel-street.js.map +0 -1
- package/out/src/tools/index.d.ts +0 -28
- package/out/src/tools/index.d.ts.map +0 -1
- package/out/src/tools/index.js +0 -28
- package/out/src/tools/index.js.map +0 -1
- package/out/src/tools/ingest-csv.d.ts.map +0 -1
- package/out/src/tools/ingest-csv.js +0 -296
- package/out/src/tools/ingest-csv.js.map +0 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +0 -61
- package/out/src/tools/jsonl-to-parquet.d.ts.map +0 -1
- package/out/src/tools/jsonl-to-parquet.js +0 -164
- package/out/src/tools/jsonl-to-parquet.js.map +0 -1
- package/out/src/tools/lint-shard-vocab.d.ts +0 -90
- package/out/src/tools/lint-shard-vocab.d.ts.map +0 -1
- package/out/src/tools/lint-shard-vocab.js +0 -281
- package/out/src/tools/lint-shard-vocab.js.map +0 -1
- package/out/src/tools/lint-shard.d.ts +0 -93
- package/out/src/tools/lint-shard.d.ts.map +0 -1
- package/out/src/tools/lint-shard.js +0 -354
- package/out/src/tools/lint-shard.js.map +0 -1
- package/out/src/tools/overlay-manifest.d.ts +0 -39
- package/out/src/tools/overlay-manifest.d.ts.map +0 -1
- package/out/src/tools/overlay-manifest.js +0 -98
- package/out/src/tools/overlay-manifest.js.map +0 -1
- package/out/src/tools/overture-subvenue.d.ts +0 -112
- package/out/src/tools/overture-subvenue.d.ts.map +0 -1
- package/out/src/tools/overture-subvenue.js +0 -144
- package/out/src/tools/overture-subvenue.js.map +0 -1
- package/out/src/tools/postcode-triples.d.ts +0 -175
- package/out/src/tools/postcode-triples.d.ts.map +0 -1
- package/out/src/tools/postcode-triples.js +0 -304
- package/out/src/tools/postcode-triples.js.map +0 -1
- package/out/src/tools/shard-kryptonite.d.ts +0 -32
- package/out/src/tools/shard-kryptonite.d.ts.map +0 -1
- package/out/src/tools/shard-kryptonite.js +0 -100
- package/out/src/tools/shard-kryptonite.js.map +0 -1
- package/out/src/tools/shard-translit.d.ts +0 -51
- package/out/src/tools/shard-translit.d.ts.map +0 -1
- package/out/src/tools/shard-translit.js +0 -175
- package/out/src/tools/shard-translit.js.map +0 -1
- package/out/src/tools/sub-venue/harvest.d.ts +0 -100
- package/out/src/tools/sub-venue/harvest.d.ts.map +0 -1
- package/out/src/tools/sub-venue/harvest.js +0 -168
- package/out/src/tools/sub-venue/harvest.js.map +0 -1
- package/out/src/tools/sub-venue/head-nouns.d.ts +0 -50
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +0 -1
- package/out/src/tools/sub-venue/head-nouns.js +0 -210
- package/out/src/tools/sub-venue/head-nouns.js.map +0 -1
- package/out/src/tools/sub-venue/surfaces.d.ts +0 -45
- package/out/src/tools/sub-venue/surfaces.d.ts.map +0 -1
- package/out/src/tools/sub-venue/surfaces.js +0 -77
- package/out/src/tools/sub-venue/surfaces.js.map +0 -1
- package/out/src/tools/sub-venue/table.d.ts +0 -229
- package/out/src/tools/sub-venue/table.d.ts.map +0 -1
- package/out/src/tools/sub-venue/table.js +0 -108
- package/out/src/tools/sub-venue/table.js.map +0 -1
- package/out/src/tools/sub-venue/wikidata.d.ts +0 -26
- package/out/src/tools/sub-venue/wikidata.d.ts.map +0 -1
- package/out/src/tools/sub-venue/wikidata.js +0 -62
- package/out/src/tools/sub-venue/wikidata.js.map +0 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +0 -156
- package/out/src/tools/sub-venue-lexicon.d.ts.map +0 -1
- package/out/src/tools/sub-venue-lexicon.js +0 -282
- package/out/src/tools/sub-venue-lexicon.js.map +0 -1
- package/out/src/tools/sub-venue-promotions.d.ts +0 -94
- package/out/src/tools/sub-venue-promotions.d.ts.map +0 -1
- package/out/src/tools/sub-venue-promotions.js +0 -266
- package/out/src/tools/sub-venue-promotions.js.map +0 -1
- package/out/src/types.d.ts +0 -221
- package/out/src/types.d.ts.map +0 -1
- package/out/src/types.js.map +0 -1
- package/out/src/utils/align.d.ts +0 -93
- package/out/src/utils/align.d.ts.map +0 -1
- package/out/src/utils/align.js +0 -234
- package/out/src/utils/align.js.map +0 -1
- package/out/src/utils/golden.d.ts +0 -69
- package/out/src/utils/golden.d.ts.map +0 -1
- package/out/src/utils/golden.js +0 -127
- package/out/src/utils/golden.js.map +0 -1
- package/out/src/utils/index.d.ts +0 -14
- package/out/src/utils/index.d.ts.map +0 -1
- package/out/src/utils/index.js +0 -14
- package/out/src/utils/index.js.map +0 -1
- package/out/src/utils/license.d.ts.map +0 -1
- package/out/src/utils/license.js.map +0 -1
- package/out/src/utils/parquet.d.ts +0 -162
- package/out/src/utils/parquet.d.ts.map +0 -1
- package/out/src/utils/parquet.js +0 -262
- package/out/src/utils/parquet.js.map +0 -1
- package/out/src/utils/split.d.ts +0 -127
- package/out/src/utils/split.d.ts.map +0 -1
- package/out/src/utils/split.js +0 -204
- package/out/src/utils/split.js.map +0 -1
- package/out/src/utils/tokenize.d.ts +0 -56
- package/out/src/utils/tokenize.d.ts.map +0 -1
- package/out/src/utils/tokenize.js +0 -48
- package/out/src/utils/tokenize.js.map +0 -1
- package/out/src/utils/wof-json.d.ts +0 -109
- package/out/src/utils/wof-json.d.ts.map +0 -1
- package/out/src/utils/wof-json.js +0 -173
- package/out/src/utils/wof-json.js.map +0 -1
- package/src/adapters/ban/adapter.ts +0 -191
- package/src/adapters/ban/street-decompose.ts +0 -96
- package/src/adapters/fcc-bdc/adapter.ts +0 -174
- package/src/adapters/geonames/adapter.ts +0 -200
- package/src/adapters/geonames-postal/adapter.ts +0 -119
- package/src/adapters/gnaf/adapter.ts +0 -149
- package/src/adapters/gnaf/assemble.ts +0 -226
- package/src/adapters/index.ts +0 -149
- package/src/adapters/openaddresses/adapter.ts +0 -251
- package/src/adapters/overture/adapter.ts +0 -162
- package/src/adapters/state-hi-schools/adapter.ts +0 -164
- package/src/adapters/synth-po-box/adapter.ts +0 -204
- package/src/adapters/tiger/adapter.ts +0 -253
- package/src/adapters/tiger/street-decompose.ts +0 -111
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +0 -183
- package/src/adapters/usgov-imls-pls/adapter.ts +0 -143
- package/src/adapters/usgov-irs-bmf/adapter.ts +0 -163
- package/src/adapters/usgov-nad/adapter.ts +0 -348
- package/src/adapters/usgov-nppes/adapter.ts +0 -159
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +0 -204
- package/src/adapters/utils/index.ts +0 -233
- package/src/adapters/wof-admin-jp/adapter.ts +0 -216
- package/src/adapters/wof-admin-json/adapter.ts +0 -256
- package/src/adapters/wof-json-rows.ts +0 -79
- package/src/adapters/wof-postalcode-json/adapter.ts +0 -183
- package/src/build.ts +0 -386
- package/src/index.ts +0 -14
- package/src/name-prone-us-suffixes.ts +0 -12
- package/src/parquet-wrapper/index.ts +0 -9
- package/src/parquet-wrapper/reader.ts +0 -97
- package/src/parquet-wrapper/schema.ts +0 -88
- package/src/parquet-wrapper/writer.ts +0 -110
- package/src/runner.ts +0 -317
- package/src/shard-recipes/anchor-absorption.ts +0 -103
- package/src/shard-recipes/bare-country.ts +0 -95
- package/src/shard-recipes/boundary-stress.ts +0 -101
- package/src/shard-recipes/country-balanced.ts +0 -524
- package/src/shard-recipes/cz-pcfirst-preposition.ts +0 -111
- package/src/shard-recipes/fr-admin-split.ts +0 -239
- package/src/shard-recipes/fr-bare-street.ts +0 -170
- package/src/shard-recipes/fr-fragment.ts +0 -332
- package/src/shard-recipes/fr-lieudit.ts +0 -302
- package/src/shard-recipes/fr-order.ts +0 -258
- package/src/shard-recipes/german.ts +0 -170
- package/src/shard-recipes/house-venue.ts +0 -88
- package/src/shard-recipes/index.ts +0 -95
- package/src/shard-recipes/intersection.ts +0 -630
- package/src/shard-recipes/locale.ts +0 -599
- package/src/shard-recipes/nl-postcode.ts +0 -115
- package/src/shard-recipes/no-fragment.ts +0 -219
- package/src/shard-recipes/no-street-led.ts +0 -143
- package/src/shard-recipes/no-street.ts +0 -86
- package/src/shard-recipes/po-box-cedex.ts +0 -920
- package/src/shard-recipes/po-box.ts +0 -119
- package/src/shard-recipes/reviewed-postcode-tail.ts +0 -189
- package/src/shard-recipes/scaffold.ts +0 -360
- package/src/shard-recipes/si-bare-village.ts +0 -97
- package/src/shard-recipes/street-affix.ts +0 -826
- package/src/shard-recipes/street-bare.ts +0 -101
- package/src/shard-recipes/street.ts +0 -82
- package/src/shard-recipes/sub-venue-sources.ts +0 -885
- package/src/shard-recipes/sub-venue.ts +0 -984
- package/src/shard-recipes/trailing-region.ts +0 -183
- package/src/shard-recipes/unit.ts +0 -321
- package/src/synthesizers/anchor-absorption.ts +0 -290
- package/src/synthesizers/boundary-stress.ts +0 -531
- package/src/synthesizers/german.ts +0 -251
- package/src/synthesizers/house-venue.ts +0 -330
- package/src/synthesizers/intersection.ts +0 -232
- package/src/synthesizers/no-street.ts +0 -349
- package/src/synthesizers/po-box.ts +0 -339
- package/src/synthesizers/street.ts +0 -274
- package/src/synthesizers/utils.ts +0 -903
- package/src/tools/align-shard.ts +0 -70
- package/src/tools/audit.ts +0 -399
- package/src/tools/corpus-stats.ts +0 -184
- package/src/tools/fetch/ban.ts +0 -290
- package/src/tools/fetch/download.ts +0 -176
- package/src/tools/fetch/geonames-dump.ts +0 -267
- package/src/tools/fetch/geonames-postal.ts +0 -180
- package/src/tools/fetch/hrsa.ts +0 -68
- package/src/tools/fetch/imls-pls.ts +0 -158
- package/src/tools/fetch/index.ts +0 -147
- package/src/tools/fetch/nad.ts +0 -342
- package/src/tools/fetch/nppes.ts +0 -174
- package/src/tools/fetch/openaddresses.ts +0 -404
- package/src/tools/fetch/ourairports.ts +0 -171
- package/src/tools/fetch/ppd.ts +0 -137
- package/src/tools/fetch/state-hi-schools.ts +0 -208
- package/src/tools/fetch/state-sources.ts +0 -155
- package/src/tools/fetch/tiger-full.ts +0 -354
- package/src/tools/fetch/wikidata-subvenue.ts +0 -386
- package/src/tools/golden-expand.ts +0 -584
- package/src/tools/golden-promote.ts +0 -353
- package/src/tools/golden-relabel-street.ts +0 -685
- package/src/tools/index.ts +0 -28
- package/src/tools/ingest-csv.ts +0 -422
- package/src/tools/jsonl-to-parquet.ts +0 -221
- package/src/tools/lint-shard-vocab.ts +0 -409
- package/src/tools/lint-shard.ts +0 -543
- package/src/tools/overlay-manifest.ts +0 -157
- package/src/tools/overture-subvenue.ts +0 -215
- package/src/tools/postcode-triples.ts +0 -375
- package/src/tools/shard-kryptonite.ts +0 -139
- package/src/tools/shard-translit.ts +0 -245
- package/src/tools/sub-venue/harvest.ts +0 -235
- package/src/tools/sub-venue/head-nouns.ts +0 -242
- package/src/tools/sub-venue/surfaces.ts +0 -95
- package/src/tools/sub-venue/table.ts +0 -281
- package/src/tools/sub-venue/wikidata.ts +0 -85
- package/src/tools/sub-venue-lexicon.ts +0 -431
- package/src/tools/sub-venue-promotions.ts +0 -330
- package/src/types.ts +0 -247
- package/src/utils/align.ts +0 -331
- package/src/utils/golden.ts +0 -176
- package/src/utils/index.ts +0 -14
- package/src/utils/parquet.ts +0 -391
- package/src/utils/split.ts +0 -279
- package/src/utils/tokenize.ts +0 -78
- package/src/utils/wof-json.ts +0 -244
- /package/{src → lib}/adapters/state-ia-contractors/adapter.ts +0 -0
- /package/{src → lib}/adapters/state-ny-notaries/adapter.ts +0 -0
- /package/{src → lib}/adapters/state-tx-notaries/adapter.ts +0 -0
- /package/{src → lib}/codex/us-fips-state.ts +0 -0
- /package/{src → lib}/tools/gb-title-case.ts +0 -0
- /package/{src → lib}/utils/license.ts +0 -0
- /package/out/{src/adapters → adapters}/ban/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/ban/street-decompose.d.ts +0 -0
- /package/out/{src/adapters → adapters}/fcc-bdc/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/geonames/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/openaddresses/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/state-ia-contractors/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/state-ia-contractors/adapter.js +0 -0
- /package/out/{src/adapters → adapters}/state-ny-notaries/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/state-ny-notaries/adapter.js +0 -0
- /package/out/{src/adapters → adapters}/state-tx-notaries/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/state-tx-notaries/adapter.js +0 -0
- /package/out/{src/adapters → adapters}/synth-po-box/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/tiger/street-decompose.d.ts +0 -0
- /package/out/{src/adapters → adapters}/usgov-nppes/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/usgov-samhsa-treatment-locator/adapter.d.ts +0 -0
- /package/out/{src/adapters → adapters}/wof-admin-jp/adapter.d.ts +0 -0
- /package/out/{src/codex → codex}/us-fips-state.d.ts +0 -0
- /package/out/{src/codex → codex}/us-fips-state.js +0 -0
- /package/out/{src/tools → tools}/fetch/ppd.d.ts +0 -0
- /package/out/{src/tools → tools}/gb-title-case.d.ts +0 -0
- /package/out/{src/tools → tools}/gb-title-case.js +0 -0
- /package/out/{src/tools → tools}/golden-promote.d.ts +0 -0
- /package/out/{src/tools → tools}/ingest-csv.d.ts +0 -0
- /package/out/{src/types.js → types.js} +0 -0
- /package/out/{src/utils → utils}/license.d.ts +0 -0
- /package/out/{src/utils → utils}/license.js +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../../src/synthesizers/utils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,EACN,+BAA+B,EAC/B,iCAAiC,EACjC,SAAS,EACT,sBAAsB,EACtB,mBAAmB,GACnB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,QAAQ,EAAE,oBAAoB,EAAsB,MAAM,cAAc,CAAA;AAEjF,OAAO,EAAE,mBAAmB,EAAkB,MAAM,sBAAsB,CAAA;AAU1E;;GAEG;AACH,SAAS,gBAAgB,CACxB,MAAoB,EACpB,MAAc,EACd,MAAc,EACd,aAA4B;IAE5B,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,EAAE,cAAc,IAAI,MAAM,CAAC,SAAS,CAAA;IAE/D,OAAO;QACN,GAAG,MAAM;QACT,GAAG,EAAE,MAAM;QACX,UAAU,EAAE,aAAa;QACzB,SAAS,EAAE,GAAG,MAAM,CAAC,SAAS,IAAI,MAAM,EAAE;QAC1C,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE;KACzC,CAAA;AACF,CAAC;AAED,8EAA8E;AAC9E,gCAAgC;AAChC,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACnC,MAAM,YAAY,GAAkB,EAAE,CAAA;IAEtC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,YAAY,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;QAClD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,KAAK,EAAE,YAAY,CAAC,CAAA;AAChE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,SAAS,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC9C,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAA;IACrC,MAAM,cAAc,GAAkB,EAAE,CAAA;IAExC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,cAAc,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAA;QACpD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,UAAU,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACvC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEzE,OAAO,gBAAgB,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,CAAA;AAC3E,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IACnC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;IAC5C,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,aAAa,CAAC,CAAiB,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAC3D,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,MAAM,QAAQ,GAAG,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAEtC,IAAI,QAAQ,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,aAAa,GAAkB,EAAE,CAAA;IAEvC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;QACrD,IAAI,CAAC,EAAE,CAAC;YACP,aAAa,CAAC,CAAiB,CAAC,GAAG,YAAY,CAAC,CAAC,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,QAAQ,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,SAAS,YAAY,CAAC,CAAS;IAC9B,OAAO,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,UAAU,CAAC,SAAS,EAAE,EAAE,CAAC,CAAA;AACpD,CAAC;AAED,8EAA8E;AAC9E,kGAAkG;AAClG,kGAAkG;AAClG,+DAA+D;AAE/D;;GAEG;AACH,MAAM,gBAAgB,GAA2B;IAChD,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,OAAO;IACV,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,KAAK;IACR,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,IAAI;IACP,CAAC,EAAE,IAAI;IACP,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,QAAQ;IACX,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,MAAM;IACT,CAAC,EAAE,KAAK;CACR,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,GAAG,2BAA2B,CAAA;AAE9C;;GAEG;AACH,SAAS,UAAU,CAAC,CAAS;IAC5B,IAAI,CAAC,GAAG,IAAI,CAAA;IAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;IAC3C,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,CAAA;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,MAAM,UAAU,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC/C,MAAM,GAAG,GAAG,UAAU,CAAC,UAAU,CAAC,GAAG,GAAG,CAAC,SAAS,OAAO,CAAC,CAAC,CAAA;IAE3D,iGAAiG;IACjG,mGAAmG;IACnG,kGAAkG;IAClG,qGAAqG;IACrG,MAAM,MAAM,GAAG,CAAC,MAAc,EAAU,EAAE;QACzC,IAAI,CAAC,GAAG,CAAC,CAAA;QAET,KAAK,IAAI,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9F,CAAC,EAAE,CAAA;QACJ,CAAC;QAED,OAAO,CAAC,CAAA;IACT,CAAC,CAAA;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;IAE9D,MAAM,QAAQ,GAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAmC,CAAC,MAAM,CACxF,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CACtG,CAAA;IAED,IAAI,CAAC,QAAQ,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACjC,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAE,CAAA;IACnE,0GAA0G;IAC1G,MAAM,SAAS,GAAa,EAAE,CAAA;IAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE;QACxC,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClB,CAAC;IAEF,IAAI,CAAC,SAAS,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAE,CAAA;IAC1D,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;IACpB,IAAI,KAAa,CAAA;IAEjB,IAAI,GAAG,EAAE,GAAG,GAAG,EAAE,CAAC;QACjB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAE1B,IAAI,EAAE,KAAK,IAAI;YAAE,OAAO,IAAI,CAAA,CAAC,qCAAqC;QAClE,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC3D,CAAC;SAAM,CAAC;QACP,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAA;QAC9B,MAAM,GAAG,GAAG,gBAAgB,CAAC,KAAK,CAAC,CAAA;QAEnC,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAA;QACrB,MAAM,GAAG,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAE,CAAA;QAChD,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,KAAK,KAAK,KAAK;QAAE,OAAO,IAAI,CAAA;IAChC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;IAE5C,+DAA+D;IAC/D,OAAO,gBAAgB,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;AACzF,CAAC,CAAA;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,kBAAkB,GAA2B;IAClD,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,QAAQ,EAAE,IAAI;IACd,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,OAAO,EAAE,IAAI;IACb,MAAM,EAAE,IAAI;IACZ,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,IAAI,EAAE,IAAI;IACV,MAAM,EAAE,IAAI;IACZ,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,QAAQ,EAAE,IAAI;IACd,aAAa,EAAE,IAAI;IACnB,QAAQ,EAAE,IAAI;IACd,SAAS,EAAE,IAAI;IACf,WAAW,EAAE,IAAI;IACjB,QAAQ,EAAE,IAAI;IACd,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,eAAe,EAAE,IAAI;IACrB,YAAY,EAAE,IAAI;IAClB,YAAY,EAAE,IAAI;IAClB,UAAU,EAAE,IAAI;IAChB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,IAAI,EAAE,IAAI;IACV,QAAQ,EAAE,IAAI;IACd,MAAM,EAAE,IAAI;IACZ,YAAY,EAAE,IAAI;IAClB,cAAc,EAAE,IAAI;IACpB,gBAAgB,EAAE,IAAI;IACtB,cAAc,EAAE,IAAI;IACpB,SAAS,EAAE,IAAI;IACf,KAAK,EAAE,IAAI;IACX,IAAI,EAAE,IAAI;IACV,OAAO,EAAE,IAAI;IACb,QAAQ,EAAE,IAAI;IACd,UAAU,EAAE,IAAI;IAChB,eAAe,EAAE,IAAI;IACrB,SAAS,EAAE,IAAI;IACf,OAAO,EAAE,IAAI;IACb,sBAAsB,EAAE,IAAI;CAC5B,CAAA;AAED,MAAM,kBAAkB,GAA2B,MAAM,CAAC,WAAW,CACpE,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAC1D,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,WAAW,GAAiB,CAAC,GAAG,EAAE,EAAE;IAChD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,wFAAwF;IACxF,wCAAwC;IACxC,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAE7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IAExE,OAAO,gBAAgB,CAAC,GAAG,EAAE,cAAc,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACpE,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAiB,CAAC,GAAG,EAAE,EAAE;IACpD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,IAAI,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAA;IAEvC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,CAAA;IAE7C,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,MAAM,KAAK,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,CAAA;IAC3E,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;IAExE,OAAO,gBAAgB,CAAC,GAAG,EAAE,kBAAkB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACxE,CAAC,CAAA;AAED,MAAM,wBAAwB,GAA2B;IACxD,KAAK,EAAE,GAAG;IACV,KAAK,EAAE,GAAG;IACV,IAAI,EAAE,GAAG;IACT,IAAI,EAAE,GAAG;IACT,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;IACf,SAAS,EAAE,IAAI;CACf,CAAA;AAED,MAAM,wBAAwB,GAA2B,MAAM,CAAC,WAAW,CAC1E,MAAM,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAChE,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACtD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAE1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAChB,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAAC,4BAA4B,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;QAEpG,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAChF,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,oBAAoB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC1E,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,qBAAqB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC1D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,WAAW,GAAmB,CAAC,QAAQ,EAAE,eAAe,EAAE,eAAe,CAAC,CAAA;IAChF,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,MAAM,GAAG,GAAG,CAAC,GAAG,CAAA;IACpB,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAE1D,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,CAAC,GAAG,aAAa,CAAC,GAAG,CAAC,CAAA;QAE5B,IAAI,CAAC,CAAC;YAAE,SAAQ;QAEhB,MAAM,QAAQ,GAAG,CAAC,CAAC,UAAU,CAC5B,sEAAsE,EACtE,CAAC,CAAC,EAAE,EAAE,CAAC,wBAAwB,CAAC,CAAC,CAAC,IAAI,CAAC,CACvC,CAAA;QAED,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACpB,aAAa,CAAC,GAAG,CAAC,GAAG,QAAQ,CAAA;YAC7B,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,QAAQ,CAAC,CAAA;YAChF,OAAO,GAAG,IAAI,CAAA;QACf,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,OAAO,gBAAgB,CAAC,GAAG,EAAE,wBAAwB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC9E,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC3D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,SAAS,GAAG,+BAA+B,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAElF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAE7F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,6BAA6B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACnF,CAAC,CAAA;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACvD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IAEpC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,CAAA;IACxB,MAAM,KAAK,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAA;IAEzC,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAExD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,MAAM,EAAE,CAAA;IAElF,IAAI,SAAS,KAAK,MAAM;QAAE,OAAO,IAAI,CAAA;IAErC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,EAAE,CAAA;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,SAAS,CAAC,CAAA;IAE7F,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,yBAAyB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AAC/E,CAAC,CAAA;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAiB,CAAC,GAAG,EAAE,EAAE;IAC7D,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAA;IAEhC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,SAAS,GAAG,iCAAiC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;IACpE,MAAM,MAAM,GAAG,SAAS,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAElD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAE9D,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAEjC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;IACzE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAEzF,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,+BAA+B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACrF,CAAC,CAAA;AAED;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACzD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,CAAA;IAEhC,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAE1C,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAA;IAEvB,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;IAExD,IAAI,MAAM,KAAK,KAAK,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzC,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAA;IAE9D,IAAI,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IAEjC,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,IAAI,EAAE,OAAO,EAAE,CAAA;IACzE,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,MAAM,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;IAEzF,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,2BAA2B,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACjF,CAAC,CAAA;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAiB,CAAC,GAAG,EAAE,EAAE;IACrD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,QAAQ,CAAA;IAExC,IAAI,CAAC,QAAQ,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC;QAAE,OAAO,IAAI,CAAA;IAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;IACxC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAEhD,IAAI,MAAM,KAAK,GAAG,CAAC,GAAG;QAAE,OAAO,IAAI,CAAA;IAEnC,OAAO,gBAAgB,CAAC,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;AACrG,CAAC,CAAA;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiB,CAAC,GAAG,EAAE,EAAE;IAClD,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,IAAI,CAAA;IACrC,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,sBAAsB,CAAA;IAEtD,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAA;IAC1B,MAAM,aAAa,GAAkB,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAC1D,OAAO,aAAa,CAAC,sBAAsB,CAAA;IAC3C,+DAA+D;IAC/D,MAAM,EAAE,GAAG,IAAI,MAAM,CAAC,OAAO,WAAW,CAAC,QAAQ,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAE9D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClC,MAAM,MAAM,GAAG,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;IAEtE,OAAO,gBAAgB,CAAC,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,aAAa,CAAC,CAAA;AACtE,CAAC,CAAA;AAED,8EAA8E;AAC9E,8BAA8B;AAC9B,8EAA8E;AAE9E;;GAEG;AACH,MAAM,CAAC,MAAM,aAAa,GAAiC;IAC1D,YAAY,EAAE,SAAS;IACvB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,UAAU;IACzB,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,cAAc,EAAE,WAAW;IAC3B,kBAAkB,EAAE,eAAe;IACnC,oBAAoB,EAAE,iBAAiB;IACvC,wBAAwB,EAAE,qBAAqB;IAC/C,6BAA6B,EAAE,sBAAsB;IACrD,yBAAyB,EAAE,kBAAkB;IAC7C,+BAA+B,EAAE,wBAAwB;IACzD,2BAA2B,EAAE,oBAAoB;IACjD,qBAAqB,EAAE,gBAAgB;IACvC,gBAAgB,EAAE,aAAa;IAC/B,aAAa,EAAE,UAAU;CACzB,CAAA;AAED;;GAEG;AACH,MAAM,UAAU,8BAA8B,CAAC,OAAe;IAC7D,6FAA6F;IAC7F,kGAAkG;IAClG,gGAAgG;IAChG,iGAAiG;IACjG,4FAA4F;IAC5F,MAAM,SAAS,GAAG,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,CAAC,CAAA;IAEjE,QAAQ,OAAO,EAAE,CAAC;QACjB,KAAK,IAAI;YACR,OAAO;gBACN,GAAG,SAAS;gBACZ,WAAW;gBACX,eAAe;gBACf,iBAAiB;gBACjB,qBAAqB;gBACrB,sBAAsB;gBACtB,kBAAkB;gBAClB,wBAAwB;gBACxB,oBAAoB;gBACpB,gBAAgB;aAChB,CAAA;QACF,KAAK,IAAI;YACR,OAAO,CAAC,GAAG,SAAS,EAAE,WAAW,EAAE,aAAa,CAAC,CAAA;QAClD;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,SAAS,CAAC,CAAC,aAAa,CAC7B,GAAiB,EACjB,gBAAyC,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC;IAEpF,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QACjC,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,CAAA;QAEpB,IAAI,GAAG,EAAE,CAAC;YACT,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;AACF,CAAC;AAED,SAAS,WAAW,CAAC,CAAS;IAC7B,OAAO,CAAC,CAAC,UAAU,CAAC,qBAAqB,EAAE,MAAM,CAAC,CAAA;AACnD,CAAC;AAoED;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,MAAM,UAAU,qBAAqB,CACpC,KAAa,EACb,OAAqB,EACrB,OAAkC;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAA;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IAE5D,MAAM,YAAY,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAEjC,IAAI,CAAC,YAAY,EAAE,CAAC;QACnB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,CAAA;IAC7E,CAAC;IAED,6FAA6F;IAC7F,+EAA+E;IAC/E,IAAI,YAAY,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,YAAY,EAAE,CAAC;QACpD,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,EAAE,CAAA;IAC/E,CAAC;IAED,MAAM,cAAc,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,CAAA;IAEvD,IAAI,cAAc,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QACvC,kFAAkF;QAClF,kFAAkF;QAClF,+BAA+B;QAC/B,OAAO;YACN,IAAI,EAAE,aAAa;YACnB,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,mBAAmB,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE;SAC7E,CAAA;IACF,CAAC;IAED,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;IAEpD,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,CAAC;QACzB,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,EAAE,CAAA;IACjF,CAAC;IAED,MAAM,WAAW,GAAe,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAA;IAE5F,MAAM,MAAM,GAAa,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAC1F,MAAM,MAAM,GAAe,CAAC,GAAG,WAAW,EAAE,GAAG,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;IAEzE,MAAM,WAAW,GAAG,GAAG,YAAY,GAAG,SAAS,GAAG,OAAO,CAAC,GAAG,EAAE,CAAA;IAE/D,MAAM,kBAAkB,GAAG;QAC1B,KAAK,EAAE,YAAY;QACnB,GAAG,OAAO,CAAC,UAAU;KACrB,CAAA;IAED,8FAA8F;IAC9F,gGAAgG;IAChG,4FAA4F;IAC5F,2FAA2F;IAC3F,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,cAAc,CAAC,GAAG,CAAA;IAEhG,IAAI,UAAU,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CACd,iFAAiF;YAChF,WAAW,OAAO,CAAC,MAAM,eAAe,OAAO,CAAC,SAAS,kCAAkC,CAC5F,CAAA;IACF,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,CAAA;IAErD,MAAM,KAAK,GAAoB;QAC9B,EAAE,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,YAAY,CAAC,MAAM,EAAE;QACpD,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAE,GAAG,MAAM,EAAE,CAAC,CAAC;KAClG,CAAA;IAED,MAAM,YAAY,GAAG,OAAO,CAAC,KAAK,EAAE,cAAc,IAAI,OAAO,CAAC,SAAS,CAAA;IACvE,MAAM,MAAM,GAAG,WAAW,OAAO,CAAC,OAAO,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAe;QAC5B,GAAG,EAAE,WAAW;QAChB,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,SAAS,EAAE,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,EAAE;QAC3C,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,OAAO,EAAE,OAAO,CAAC,OAAO;QACxB,KAAK,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,YAAY,EAAE;QAC/C,MAAM;QACN,MAAM;QACN,WAAW,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACtC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAClC,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAClC,CAAA;IAED,oBAAoB,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAA;IAErC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;AAC1C,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"align-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AAgCA,MAAM,WAAW,iBAAiB;IACjC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,aAAa,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,IAAI,CAAC,CA+BhF"}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
import { createWriteStream } from "node:fs";
|
|
3
|
-
import { JSONSpliterator } from "spliterator";
|
|
4
|
-
/**
|
|
5
|
-
* Re-emit a CANONICAL jsonl ({raw, components, country, source, ...}) as a LABELED jsonl in the CURRENT align format,
|
|
6
|
-
* by running every row through `alignRow` (corpus/src/align.ts).
|
|
7
|
-
*
|
|
8
|
-
* ## Why this exists
|
|
9
|
-
*
|
|
10
|
-
* Most synthetic shards are GENERATED on demand by a `build-*-shard` recipe (parametrized by --count), so re-emitting
|
|
11
|
-
* them in a new label format is just a re-run. A few shards are FIXED corpora with a hand/DeepSeek-authored canonical
|
|
12
|
-
* source that is never regenerated — notably `deepseek-kryptonite` (the adversarial hard-case set) and the
|
|
13
|
-
* `deepseek-translit-*` variants. Their committed parquets carry whatever label format was current when they were first
|
|
14
|
-
* built.
|
|
15
|
-
*
|
|
16
|
-
* When the corpus label format changes (the v0.5.0 char-offset triple, #519), those fixed shards must be RE-ALIGNED,
|
|
17
|
-
* not regenerated — feed the canonical source back through the same `alignRow` the from-source build uses, so the spans
|
|
18
|
-
* land in the new format with zero drift. That is exactly what this does: canonical jsonl in → labeled jsonl out, one
|
|
19
|
-
* `alignRow` per row, quarantine on miss.
|
|
20
|
-
*
|
|
21
|
-
* It is the uniform counterpart to corpus/src/tools/shard-kryptonite.ts (which couples to a base manifest and writes
|
|
22
|
-
* parquet directly). Output goes to jsonl so it joins the SAME jsonl-to-parquet path every other overlay shard uses.
|
|
23
|
-
*
|
|
24
|
-
* Usage: node scripts/align-canonical-shard.ts\
|
|
25
|
-
* --input /path/canonical-kryptonite.jsonl\
|
|
26
|
-
* --output /tmp/kryptonite-labeled.jsonl\
|
|
27
|
-
* --corpus-version 0.5.0
|
|
28
|
-
*/
|
|
29
|
-
import { alignRow } from "#utils";
|
|
30
|
-
export async function alignCanonicalShard(args) {
|
|
31
|
-
// Read phase only — the write path stays on createWriteStream.
|
|
32
|
-
const outStream = createWriteStream(args.output, { encoding: "utf8" });
|
|
33
|
-
let labeled = 0;
|
|
34
|
-
let quarantined = 0;
|
|
35
|
-
const quarantineReasons = {};
|
|
36
|
-
for await (const canonical of JSONSpliterator.fromAsync(args.input)) {
|
|
37
|
-
// Stamp the target corpus version so the emitted row's provenance matches the run it joins.
|
|
38
|
-
canonical.corpus_version = args.corpusVersion;
|
|
39
|
-
const result = alignRow(canonical);
|
|
40
|
-
if (result.kind === "labeled") {
|
|
41
|
-
outStream.write(JSON.stringify(result.row) + "\n");
|
|
42
|
-
labeled++;
|
|
43
|
-
}
|
|
44
|
-
else {
|
|
45
|
-
quarantined++;
|
|
46
|
-
const r = result.row.reason ?? "unknown";
|
|
47
|
-
quarantineReasons[r] = (quarantineReasons[r] ?? 0) + 1;
|
|
48
|
-
}
|
|
49
|
-
}
|
|
50
|
-
await new Promise((res) => {
|
|
51
|
-
outStream.end(res);
|
|
52
|
-
});
|
|
53
|
-
console.error(`align-canonical-shard: ${labeled} labeled, ${quarantined} quarantined → ${args.output}\n` +
|
|
54
|
-
` quarantine reasons: ${JSON.stringify(quarantineReasons)}`);
|
|
55
|
-
}
|
|
56
|
-
//# sourceMappingURL=align-shard.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"align-shard.js","sourceRoot":"","sources":["../../../src/tools/align-shard.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AACH,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAQjC,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAuB;IAChE,+DAA+D;IAC/D,MAAM,SAAS,GAAG,iBAAiB,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACtE,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAA2B,EAAE,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,SAAS,IAAI,eAAe,CAAC,SAAS,CAAiC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrG,4FAA4F;QAC5F,SAAS,CAAC,cAAc,GAAG,IAAI,CAAC,aAAa,CAAA;QAC7C,MAAM,MAAM,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;QAElC,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;YAC/B,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;YAElD,OAAO,EAAE,CAAA;QACV,CAAC;aAAM,CAAC;YACP,WAAW,EAAE,CAAA;YACb,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,SAAS,CAAA;YACxC,iBAAiB,CAAC,CAAC,CAAC,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACvD,CAAC;IACF,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACnB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,KAAK,CACZ,0BAA0B,OAAO,aAAa,WAAW,kBAAkB,IAAI,CAAC,MAAM,IAAI;QACzF,yBAAyB,IAAI,CAAC,SAAS,CAAC,iBAAiB,CAAC,EAAE,CAC7D,CAAA;AACF,CAAC"}
|
package/out/src/tools/audit.d.ts
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* `mailwoman corpus audit` — per-source shard-count vs source_weight diagnostic.
|
|
7
|
-
*
|
|
8
|
-
* Reads a corpus dir's MANIFEST.json (or scans shards directly), counts shards per source,
|
|
9
|
-
* optionally loads a training config to pair the counts with the configured source_weights, and
|
|
10
|
-
* reports the estimated sampled-row distribution at training time.
|
|
11
|
-
*
|
|
12
|
-
* Would have caught v0.3.0's "NAD = 411/674 train shards × 2.0 weight = ~75% of sampled mix"
|
|
13
|
-
* finding before the v0.3.0 retrospective surfaced it.
|
|
14
|
-
*
|
|
15
|
-
* Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
|
|
16
|
-
*/
|
|
17
|
-
export interface AuditOpts {
|
|
18
|
-
corpusDir: string;
|
|
19
|
-
configPath?: string;
|
|
20
|
-
/**
|
|
21
|
-
* Sample at most N shards per split when counting sources. Default 100 for speed; bump to read the full set on a slow
|
|
22
|
-
* run. The first row of each shard determines its source — corpus-v0.2.0+ shards are 100% source-segregated, so a
|
|
23
|
-
* one-row read is authoritative.
|
|
24
|
-
*/
|
|
25
|
-
sampleShardCount?: number;
|
|
26
|
-
}
|
|
27
|
-
export declare function audit(opts: AuditOpts): void;
|
|
28
|
-
//# sourceMappingURL=audit.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"audit.d.ts","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAmBH,MAAM,WAAW,SAAS;IACzB,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;;;OAIG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;CACzB;AA0VD,wBAAgB,KAAK,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI,CAS3C"}
|
package/out/src/tools/audit.js
DELETED
|
@@ -1,285 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* `mailwoman corpus audit` — per-source shard-count vs source_weight diagnostic.
|
|
7
|
-
*
|
|
8
|
-
* Reads a corpus dir's MANIFEST.json (or scans shards directly), counts shards per source,
|
|
9
|
-
* optionally loads a training config to pair the counts with the configured source_weights, and
|
|
10
|
-
* reports the estimated sampled-row distribution at training time.
|
|
11
|
-
*
|
|
12
|
-
* Would have caught v0.3.0's "NAD = 411/674 train shards × 2.0 weight = ~75% of sampled mix"
|
|
13
|
-
* finding before the v0.3.0 retrospective surfaced it.
|
|
14
|
-
*
|
|
15
|
-
* Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
|
|
16
|
-
*/
|
|
17
|
-
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
|
18
|
-
import { basename, join } from "node:path";
|
|
19
|
-
import { parseJSONStrict } from "@mailwoman/core/objects";
|
|
20
|
-
import { TextSpliterator } from "spliterator";
|
|
21
|
-
/**
|
|
22
|
-
* Share of a shard one source may hold before the mix is flagged as dominated by it.
|
|
23
|
-
*/
|
|
24
|
-
const DOMINANT_SOURCE_SHARE = 0.4;
|
|
25
|
-
/**
|
|
26
|
-
* How far the leading source may outweigh the runner-up before the mix is flagged. Catches the case where no single
|
|
27
|
-
* source clears {@link DOMINANT_SOURCE_SHARE} but the distribution is still lopsided.
|
|
28
|
-
*/
|
|
29
|
-
const MAX_TOP_TO_RUNNER_UP_RATIO = 1.5;
|
|
30
|
-
/**
|
|
31
|
-
* Try parsing a training YAML's source_weights as a minimal regex-based extract. We don't pull in a YAML lib for this
|
|
32
|
-
* script — the syntax is so small that a regex over the source_weights block is sufficient + keeps the script
|
|
33
|
-
* dep-free.
|
|
34
|
-
*/
|
|
35
|
-
function parseConfig(configPath) {
|
|
36
|
-
if (!existsSync(configPath))
|
|
37
|
-
return null;
|
|
38
|
-
const weights = {};
|
|
39
|
-
let inBlock = false;
|
|
40
|
-
let blockIndent = -1;
|
|
41
|
-
for (const raw of TextSpliterator.from(readFileSync(configPath))) {
|
|
42
|
-
const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/);
|
|
43
|
-
if (sourceWeightsMatch) {
|
|
44
|
-
inBlock = true;
|
|
45
|
-
blockIndent = sourceWeightsMatch[1].length;
|
|
46
|
-
continue;
|
|
47
|
-
}
|
|
48
|
-
if (!inBlock)
|
|
49
|
-
continue;
|
|
50
|
-
// Skip blank lines and comments.
|
|
51
|
-
if (/^[\t ]*(#|$)/.test(raw))
|
|
52
|
-
continue;
|
|
53
|
-
// Lines indented MORE than `source_weights:` are entries; lines with ≤ indent end the block.
|
|
54
|
-
const indent = raw.match(/^[\t ]*/)[0].length;
|
|
55
|
-
if (indent <= blockIndent) {
|
|
56
|
-
inBlock = false;
|
|
57
|
-
continue;
|
|
58
|
-
}
|
|
59
|
-
const m = raw.match(/^[\t ]+([\w-]+):\s*([\d.]+)/);
|
|
60
|
-
if (m) {
|
|
61
|
-
weights[m[1]] = Number.parseFloat(m[2]);
|
|
62
|
-
}
|
|
63
|
-
}
|
|
64
|
-
return { sourceWeights: weights };
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Scan a corpus directory's shards (typically under <corpus_dir>/train, /val, /test) and count shards per source per
|
|
68
|
-
* split.
|
|
69
|
-
*/
|
|
70
|
-
function scanShards(corpusDir, sampleCount) {
|
|
71
|
-
const stats = { bySplit: {}, totalShards: 0, totalFiles: 0 };
|
|
72
|
-
for (const split of ["train", "val", "test"]) {
|
|
73
|
-
const splitDir = join(corpusDir, split);
|
|
74
|
-
if (!existsSync(splitDir))
|
|
75
|
-
continue;
|
|
76
|
-
const files = readdirSync(splitDir)
|
|
77
|
-
.filter((f) => f.endsWith(".parquet"))
|
|
78
|
-
.toSorted();
|
|
79
|
-
stats.totalFiles += files.length;
|
|
80
|
-
const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount));
|
|
81
|
-
const sampled = files.filter((_, i) => i % sampleEvery === 0).slice(0, sampleCount);
|
|
82
|
-
const splitMap = {};
|
|
83
|
-
// We can't read parquet without a dep, so we infer source from filenames where possible.
|
|
84
|
-
// The corpus build typically writes deterministically by source — fall back to "<unknown>"
|
|
85
|
-
// when filename gives no hint. For accurate per-source counts on real corpora, the
|
|
86
|
-
// MANIFEST.json route below is preferred.
|
|
87
|
-
for (const f of sampled) {
|
|
88
|
-
const inferred = inferSourceFromFilename(f);
|
|
89
|
-
splitMap[inferred] = (splitMap[inferred] ?? 0) + 1;
|
|
90
|
-
}
|
|
91
|
-
// Scale to estimated full-shard counts.
|
|
92
|
-
const scale = files.length / Math.max(sampled.length, 1);
|
|
93
|
-
for (const k of Object.keys(splitMap)) {
|
|
94
|
-
splitMap[k] = Math.round(splitMap[k] * scale);
|
|
95
|
-
}
|
|
96
|
-
stats.bySplit[split] = splitMap;
|
|
97
|
-
stats.totalShards += files.length;
|
|
98
|
-
}
|
|
99
|
-
return stats;
|
|
100
|
-
}
|
|
101
|
-
function inferSourceFromFilename(filename) {
|
|
102
|
-
// Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet. The latter (current
|
|
103
|
-
// build at corpus-v0.3.0) gives no source signal in the filename — see manifestScan() for the
|
|
104
|
-
// authoritative path. Return "<unknown>" so the caller flags this case.
|
|
105
|
-
const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/);
|
|
106
|
-
if (m && m[1] !== undefined)
|
|
107
|
-
return m[1];
|
|
108
|
-
return "<unknown>";
|
|
109
|
-
}
|
|
110
|
-
/**
|
|
111
|
-
* Known source name prefixes. Corpus-v0.3.0 uses these as `source_id` prefixes; matching against the longest prefix
|
|
112
|
-
* that fits a given `first_source_id` recovers the canonical source name.
|
|
113
|
-
*
|
|
114
|
-
* Order matters: longer prefixes must be tried first so `usgov-nad-...` matches `usgov-nad` rather than `usgov`. Sorted
|
|
115
|
-
* descending by length at use site.
|
|
116
|
-
*/
|
|
117
|
-
const KNOWN_SOURCE_PREFIXES = [
|
|
118
|
-
"wof-admin",
|
|
119
|
-
"wof-postalcode",
|
|
120
|
-
"ban",
|
|
121
|
-
"tiger",
|
|
122
|
-
"usgov-nad",
|
|
123
|
-
"usgov-nppes",
|
|
124
|
-
"usgov-hrsa-fqhc",
|
|
125
|
-
"usgov-imls-pls",
|
|
126
|
-
"state-ia-contractors",
|
|
127
|
-
"state-tx-notaries",
|
|
128
|
-
"state-ny-notaries",
|
|
129
|
-
"openaddresses",
|
|
130
|
-
// Synthetic adversarial sources (corpus-v0.4.0+, Thread B).
|
|
131
|
-
"deepseek-kryptonite",
|
|
132
|
-
"deepseek-translit-cyrl",
|
|
133
|
-
"deepseek-translit-jpan",
|
|
134
|
-
"deepseek-translit-hans",
|
|
135
|
-
"deepseek-translit-hang",
|
|
136
|
-
"deepseek-translit-armn",
|
|
137
|
-
];
|
|
138
|
-
/**
|
|
139
|
-
* Extract the source-name prefix from a `first_source_id` value.
|
|
140
|
-
*/
|
|
141
|
-
function sourceFromID(sourceID, knownPrefixes) {
|
|
142
|
-
// Sort longest-first so usgov-nad beats usgov, wof-admin beats wof.
|
|
143
|
-
const sorted = [...knownPrefixes].toSorted((a, b) => b.length - a.length);
|
|
144
|
-
for (const prefix of sorted) {
|
|
145
|
-
if (sourceID.startsWith(prefix + "-") || sourceID === prefix)
|
|
146
|
-
return prefix;
|
|
147
|
-
}
|
|
148
|
-
return "<unknown>";
|
|
149
|
-
}
|
|
150
|
-
/**
|
|
151
|
-
* Prefer reading MANIFEST.json when present — uses each shard's `first_source_id` + prefix matching to recover the
|
|
152
|
-
* source name. Falls back to scanShards when MANIFEST is absent.
|
|
153
|
-
*
|
|
154
|
-
* NOTE: corpus-v0.3.0 shards can mix sources (see `last_source_id` differing from `first_source_id`). The first-row
|
|
155
|
-
* source is an approximation; reading the parquet's full source column would be authoritative but requires a parquet
|
|
156
|
-
* dep. For audit purposes the first-row approximation is accurate within ~5% for the corpus-v0.3.0 shape (most shards
|
|
157
|
-
* are >95% one source).
|
|
158
|
-
*/
|
|
159
|
-
function manifestScan(corpusDir, knownPrefixes) {
|
|
160
|
-
const manifestPath = join(corpusDir, "MANIFEST.json");
|
|
161
|
-
if (!existsSync(manifestPath))
|
|
162
|
-
return null;
|
|
163
|
-
const manifest = parseJSONStrict(readFileSync(manifestPath, "utf8"));
|
|
164
|
-
if (!Array.isArray(manifest.shards))
|
|
165
|
-
return null;
|
|
166
|
-
const bySplit = {};
|
|
167
|
-
for (const shard of manifest.shards) {
|
|
168
|
-
const split = shard.split;
|
|
169
|
-
const src = shard.source ?? sourceFromID(shard.first_source_id ?? "", knownPrefixes);
|
|
170
|
-
bySplit[split] ??= {};
|
|
171
|
-
bySplit[split][src] = (bySplit[split][src] ?? 0) + 1;
|
|
172
|
-
}
|
|
173
|
-
const total = Object.values(bySplit).reduce((sum, m) => sum + Object.values(m).reduce((a, b) => a + b, 0), 0);
|
|
174
|
-
return { bySplit, totalShards: total, totalFiles: total };
|
|
175
|
-
}
|
|
176
|
-
function buildAuditRows(stats, weights) {
|
|
177
|
-
const totalShards = Object.values(stats).reduce((a, b) => a + b, 0);
|
|
178
|
-
const allSources = new Set([...Object.keys(stats), ...Object.keys(weights)]);
|
|
179
|
-
const rows = [];
|
|
180
|
-
// Compute effective sample weight: shard_count × source_weight. Sources with no weight get the
|
|
181
|
-
// "—" marker (loader skips them).
|
|
182
|
-
const sampleWeights = [];
|
|
183
|
-
for (const src of allSources) {
|
|
184
|
-
const shards = stats[src] ?? 0;
|
|
185
|
-
const weight = weights[src];
|
|
186
|
-
const effective = weight !== undefined ? shards * weight : 0;
|
|
187
|
-
sampleWeights.push([src, effective]);
|
|
188
|
-
}
|
|
189
|
-
const totalSampleWeight = sampleWeights.reduce((a, [, w]) => a + w, 0);
|
|
190
|
-
for (const src of allSources) {
|
|
191
|
-
const shards = stats[src] ?? 0;
|
|
192
|
-
const weight = weights[src] ?? "—";
|
|
193
|
-
const effective = typeof weight === "number" ? (shards * weight) / Math.max(totalSampleWeight, 1) : "—";
|
|
194
|
-
rows.push({
|
|
195
|
-
source: src,
|
|
196
|
-
shards,
|
|
197
|
-
shardPct: totalShards > 0 ? shards / totalShards : 0,
|
|
198
|
-
weight,
|
|
199
|
-
effectiveSamplePct: typeof effective === "number" ? effective : "—",
|
|
200
|
-
});
|
|
201
|
-
}
|
|
202
|
-
// Flag the dominator: empirically calibrated against the v0.3.0 → v0.4.0 retrospective.
|
|
203
|
-
// v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space dilution
|
|
204
|
-
// was responsible for the coarse-F1 regression. So flag a source as "concentration warning"
|
|
205
|
-
// when it's above 40% effective sample OR more than 1.5× the next-highest.
|
|
206
|
-
const numeric = rows.filter((r) => typeof r.effectiveSamplePct === "number");
|
|
207
|
-
numeric.sort((a, b) => b.effectiveSamplePct - a.effectiveSamplePct);
|
|
208
|
-
if (numeric.length) {
|
|
209
|
-
const top = numeric[0];
|
|
210
|
-
const next = numeric[1]?.effectiveSamplePct ?? 0;
|
|
211
|
-
if (top.effectiveSamplePct > DOMINANT_SOURCE_SHARE ||
|
|
212
|
-
(next > 0 && top.effectiveSamplePct / next > MAX_TOP_TO_RUNNER_UP_RATIO)) {
|
|
213
|
-
top.overweightFactor = next > 0 ? top.effectiveSamplePct / next : Infinity;
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
rows.sort((a, b) => b.shards - a.shards);
|
|
217
|
-
return rows;
|
|
218
|
-
}
|
|
219
|
-
function formatPct(v) {
|
|
220
|
-
if (v === "—")
|
|
221
|
-
return "—";
|
|
222
|
-
return `${(v * 100).toFixed(1)}%`;
|
|
223
|
-
}
|
|
224
|
-
function printReport(corpusDir, configPath, stats, rows) {
|
|
225
|
-
console.log(`\nCorpus audit — ${corpusDir}`);
|
|
226
|
-
if (configPath) {
|
|
227
|
-
console.log(`Config: ${configPath}`);
|
|
228
|
-
}
|
|
229
|
-
console.log(`Total shards: ${stats.totalShards}${stats.totalFiles !== stats.totalShards ? ` (${stats.totalFiles} files on disk)` : ""}`);
|
|
230
|
-
console.log("");
|
|
231
|
-
const trainStats = stats.bySplit["train"];
|
|
232
|
-
if (trainStats) {
|
|
233
|
-
const total = Object.values(trainStats).reduce((a, b) => a + b, 0);
|
|
234
|
-
console.log(`Train split: ${total} shards`);
|
|
235
|
-
console.log("");
|
|
236
|
-
const headers = ["source", "shards", "shard %", "weight", "eff. sample %"];
|
|
237
|
-
const widths = [22, 8, 10, 8, 14];
|
|
238
|
-
const fmtRow = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join(" ");
|
|
239
|
-
console.log(fmtRow(headers));
|
|
240
|
-
console.log(fmtRow(widths.map((w) => "─".repeat(w))));
|
|
241
|
-
for (const row of rows) {
|
|
242
|
-
console.log(fmtRow([
|
|
243
|
-
row.source,
|
|
244
|
-
String(row.shards),
|
|
245
|
-
formatPct(row.shardPct),
|
|
246
|
-
typeof row.weight === "number" ? row.weight.toFixed(2) : "—",
|
|
247
|
-
formatPct(row.effectiveSamplePct),
|
|
248
|
-
]));
|
|
249
|
-
}
|
|
250
|
-
console.log("");
|
|
251
|
-
const dominator = rows.find((r) => r.overweightFactor !== undefined);
|
|
252
|
-
if (dominator) {
|
|
253
|
-
const factor = dominator.overweightFactor;
|
|
254
|
-
const factorStr = factor === Infinity ? "∞" : factor?.toFixed(1);
|
|
255
|
-
console.error(`⚠ Concentration: ${dominator.source} would sample ${formatPct(dominator.effectiveSamplePct)} ` +
|
|
256
|
-
`of training rows (${factorStr}× the next-highest). ` +
|
|
257
|
-
`Past lesson: v0.3.0's NAD at ~52% caused the 21-label coarse regression. ` +
|
|
258
|
-
`Consider lowering this source's weight or boosting others.`);
|
|
259
|
-
}
|
|
260
|
-
else {
|
|
261
|
-
console.log("✓ No single-source concentration (top source < 40% effective sample AND < 1.5× next).");
|
|
262
|
-
}
|
|
263
|
-
const missingWeights = rows.filter((r) => r.weight === "—" && r.shards > 0);
|
|
264
|
-
if (missingWeights.length && configPath) {
|
|
265
|
-
console.error(`⚠ Sources present in corpus but absent from config.source_weights ` +
|
|
266
|
-
`(loader will skip them): ${missingWeights.map((r) => r.source).join(", ")}`);
|
|
267
|
-
}
|
|
268
|
-
const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.shards === 0);
|
|
269
|
-
if (orphanWeights.length) {
|
|
270
|
-
console.error(`⚠ Sources weighted in config but no shards found in corpus ` +
|
|
271
|
-
`(no-op weights): ${orphanWeights.map((r) => r.source).join(", ")}`);
|
|
272
|
-
}
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
export function audit(opts) {
|
|
276
|
-
const config = opts.configPath ? parseConfig(opts.configPath) : null;
|
|
277
|
-
// Compose the known-prefix list from both the hardcoded set and any extra names in the config
|
|
278
|
-
// (forward-compat for future adapters added before this file is updated).
|
|
279
|
-
const prefixes = [...new Set([...KNOWN_SOURCE_PREFIXES, ...Object.keys(config?.sourceWeights ?? {})])];
|
|
280
|
-
const stats = manifestScan(opts.corpusDir, prefixes) ?? scanShards(opts.corpusDir, opts.sampleShardCount ?? 100);
|
|
281
|
-
const trainStats = stats.bySplit["train"] ?? {};
|
|
282
|
-
const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {});
|
|
283
|
-
printReport(opts.corpusDir, opts.configPath, stats, rows);
|
|
284
|
-
}
|
|
285
|
-
//# sourceMappingURL=audit.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../../src/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AAC/D,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAE1C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAgCtC;;;;GAIG;AACH,SAAS,WAAW,CAAC,UAAkB;IACtC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;QAAE,OAAO,IAAI,CAAA;IACxC,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAClE,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAA;YACd,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YAE3C,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QACtC,6FAA6F;QAC7F,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE9C,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAElD,IAAI,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,SAAS,UAAU,CAAC,SAAiB,EAAE,WAAmB;IACzD,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAExE,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAEvC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;YAAE,SAAQ;QAEnC,MAAM,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC;aACjC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;aACrC,QAAQ,EAAE,CAAA;QAEZ,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,2FAA2F;QAC3F,mFAAmF;QACnF,0CAA0C;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;IAClC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,8FAA8F;IAC9F,8FAA8F;IAC9F,wEAAwE;IACxE,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,SAAS,YAAY,CAAC,SAAiB,EAAE,aAAgC;IACxE,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAErD,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,MAAM,QAAQ,GAAG,eAAe,CAE7B,YAAY,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IAEtC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QACpF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC1D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+FAA+F;IAC/F,kCAAkC;IAClC,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEvG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,MAAM;YACN,QAAQ,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,8FAA8F;IAC9F,4FAA4F;IAC5F,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAExC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,SAAiB,EAAE,UAA8B,EAAE,KAAiB,EAAE,IAAgB;IAC1G,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACvB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE3E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;QAExF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,6DAA6D;gBAC5D,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,IAAe;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACpE,8FAA8F;IAC9F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IACtG,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAA;IAChH,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}
|
|
@@ -1,44 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
|
|
7
|
-
*
|
|
8
|
-
* Reads one or more Parquet shards, builds per-(token, label) and per-(bigram, label-bigram)
|
|
9
|
-
* histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-shard.ts`
|
|
10
|
-
* as the baseline against which a new shard is compared.
|
|
11
|
-
*
|
|
12
|
-
* Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
|
|
13
|
-
* linter invocations. Re-run this script whenever the corpus changes substantially (a new
|
|
14
|
-
* mainline shard added, a source-pool re-weighted, etc.).
|
|
15
|
-
*
|
|
16
|
-
* Output schema:
|
|
17
|
-
*
|
|
18
|
-
* ```ts
|
|
19
|
-
* interface CorpusStats {
|
|
20
|
-
* row_count: number
|
|
21
|
-
* shard_paths: string[]
|
|
22
|
-
* tokens: { [token: string]: { [label: string]: number } }
|
|
23
|
-
* bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
|
|
24
|
-
* // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
|
|
25
|
-
* // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
|
|
26
|
-
* }
|
|
27
|
-
* ```
|
|
28
|
-
*
|
|
29
|
-
* Usage: node scripts/build-corpus-stats.ts\
|
|
30
|
-
* --shards <glob-pattern-or-dir>\
|
|
31
|
-
* --output <stats.json>
|
|
32
|
-
*
|
|
33
|
-
* For a quick local-corpus baseline (limited but useful for linter testing): node
|
|
34
|
-
* scripts/build-corpus-stats.ts\
|
|
35
|
-
* --shards $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
|
|
36
|
-
* --output /tmp/corpus-stats-local.json
|
|
37
|
-
*/
|
|
38
|
-
export interface CorpusStatsOptions {
|
|
39
|
-
shardsArg: string;
|
|
40
|
-
outputPath: string;
|
|
41
|
-
limitPerShard?: number;
|
|
42
|
-
}
|
|
43
|
-
export declare function buildCorpusStats(args: CorpusStatsOptions): Promise<void>;
|
|
44
|
-
//# sourceMappingURL=corpus-stats.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../../src/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAUH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAuCD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CA8F9E"}
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @copyright Sister Software
|
|
3
|
-
* @license AGPL-3.0
|
|
4
|
-
* @author Teffen Ellis, et al.
|
|
5
|
-
*
|
|
6
|
-
* Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
|
|
7
|
-
*
|
|
8
|
-
* Reads one or more Parquet shards, builds per-(token, label) and per-(bigram, label-bigram)
|
|
9
|
-
* histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-shard.ts`
|
|
10
|
-
* as the baseline against which a new shard is compared.
|
|
11
|
-
*
|
|
12
|
-
* Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
|
|
13
|
-
* linter invocations. Re-run this script whenever the corpus changes substantially (a new
|
|
14
|
-
* mainline shard added, a source-pool re-weighted, etc.).
|
|
15
|
-
*
|
|
16
|
-
* Output schema:
|
|
17
|
-
*
|
|
18
|
-
* ```ts
|
|
19
|
-
* interface CorpusStats {
|
|
20
|
-
* row_count: number
|
|
21
|
-
* shard_paths: string[]
|
|
22
|
-
* tokens: { [token: string]: { [label: string]: number } }
|
|
23
|
-
* bigrams: { [token_bigram: string]: { [label_bigram: string]: number } }
|
|
24
|
-
* // token_bigram = "tok1tok2" (US sep), label_bigram = "lab1lab2"
|
|
25
|
-
* // For memory: only keep bigrams with count >= MIN_BIGRAM_COUNT (2).
|
|
26
|
-
* }
|
|
27
|
-
* ```
|
|
28
|
-
*
|
|
29
|
-
* Usage: node scripts/build-corpus-stats.ts\
|
|
30
|
-
* --shards <glob-pattern-or-dir>\
|
|
31
|
-
* --output <stats.json>
|
|
32
|
-
*
|
|
33
|
-
* For a quick local-corpus baseline (limited but useful for linter testing): node
|
|
34
|
-
* scripts/build-corpus-stats.ts\
|
|
35
|
-
* --shards $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
|
|
36
|
-
* --output /tmp/corpus-stats-local.json
|
|
37
|
-
*/
|
|
38
|
-
import { readdirSync, statSync, writeFileSync } from "node:fs";
|
|
39
|
-
import { join } from "node:path";
|
|
40
|
-
import { ParquetReader } from "../parquet-wrapper/index.js";
|
|
41
|
-
const SEP = "";
|
|
42
|
-
const MIN_BIGRAM_COUNT = 2;
|
|
43
|
-
function discoverShards(shardsArg) {
|
|
44
|
-
const stat = statSync(shardsArg);
|
|
45
|
-
if (stat.isDirectory()) {
|
|
46
|
-
return readdirSync(shardsArg)
|
|
47
|
-
.filter((f) => f.endsWith(".parquet"))
|
|
48
|
-
.map((f) => join(shardsArg, f));
|
|
49
|
-
}
|
|
50
|
-
if (stat.isFile() && shardsArg.endsWith(".parquet"))
|
|
51
|
-
return [shardsArg];
|
|
52
|
-
// Otherwise treat as a literal path list (one per line if it's stdin-friendly).
|
|
53
|
-
return [shardsArg];
|
|
54
|
-
}
|
|
55
|
-
/**
|
|
56
|
-
* Stream a shard's `tokens`/`labels` columns.
|
|
57
|
-
*
|
|
58
|
-
* `limit` stops the iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
|
|
59
|
-
*/
|
|
60
|
-
async function* streamShardRows(shardPath, limit) {
|
|
61
|
-
await using reader = await ParquetReader.openFile(shardPath);
|
|
62
|
-
let emitted = 0;
|
|
63
|
-
for await (const row of reader.project("tokens", "labels")) {
|
|
64
|
-
if (limit !== undefined && emitted >= limit)
|
|
65
|
-
break;
|
|
66
|
-
yield row;
|
|
67
|
-
emitted++;
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
export async function buildCorpusStats(args) {
|
|
71
|
-
const shardPaths = discoverShards(args.shardsArg);
|
|
72
|
-
console.error(`Discovered ${shardPaths.length} parquet shard(s)`);
|
|
73
|
-
const tokenStats = new Map();
|
|
74
|
-
const bigramStats = new Map();
|
|
75
|
-
let totalRows = 0;
|
|
76
|
-
for (const path of shardPaths) {
|
|
77
|
-
console.error(`Reading ${path}...`);
|
|
78
|
-
const before = totalRows;
|
|
79
|
-
for await (const { tokens, labels } of streamShardRows(path, args.limitPerShard)) {
|
|
80
|
-
totalRows++;
|
|
81
|
-
if (tokens.length !== labels.length)
|
|
82
|
-
continue;
|
|
83
|
-
// skip malformed
|
|
84
|
-
for (let i = 0; i < tokens.length; i++) {
|
|
85
|
-
const tk = tokens[i];
|
|
86
|
-
const lb = labels[i];
|
|
87
|
-
let labelMap = tokenStats.get(tk);
|
|
88
|
-
if (!labelMap) {
|
|
89
|
-
labelMap = new Map();
|
|
90
|
-
tokenStats.set(tk, labelMap);
|
|
91
|
-
}
|
|
92
|
-
labelMap.set(lb, (labelMap.get(lb) ?? 0) + 1);
|
|
93
|
-
if (i + 1 < tokens.length) {
|
|
94
|
-
const bigramKey = tk + SEP + tokens[i + 1];
|
|
95
|
-
const bigramLabel = lb + SEP + labels[i + 1];
|
|
96
|
-
let bMap = bigramStats.get(bigramKey);
|
|
97
|
-
if (!bMap) {
|
|
98
|
-
bMap = new Map();
|
|
99
|
-
bigramStats.set(bigramKey, bMap);
|
|
100
|
-
}
|
|
101
|
-
bMap.set(bigramLabel, (bMap.get(bigramLabel) ?? 0) + 1);
|
|
102
|
-
}
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
console.error(` ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`);
|
|
106
|
-
}
|
|
107
|
-
// Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
|
|
108
|
-
// stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
|
|
109
|
-
// detection.
|
|
110
|
-
let prunedBigrams = 0;
|
|
111
|
-
for (const [k, labelMap] of bigramStats) {
|
|
112
|
-
let total = 0;
|
|
113
|
-
for (const v of labelMap.values()) {
|
|
114
|
-
total += v;
|
|
115
|
-
}
|
|
116
|
-
if (total < MIN_BIGRAM_COUNT) {
|
|
117
|
-
bigramStats.delete(k);
|
|
118
|
-
prunedBigrams++;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`);
|
|
122
|
-
const out = {
|
|
123
|
-
row_count: totalRows,
|
|
124
|
-
shard_paths: shardPaths,
|
|
125
|
-
tokens: {},
|
|
126
|
-
bigrams: {},
|
|
127
|
-
};
|
|
128
|
-
for (const [tk, labelMap] of tokenStats) {
|
|
129
|
-
out.tokens[tk] = Object.fromEntries(labelMap);
|
|
130
|
-
}
|
|
131
|
-
for (const [k, labelMap] of bigramStats) {
|
|
132
|
-
out.bigrams[k] = Object.fromEntries(labelMap);
|
|
133
|
-
}
|
|
134
|
-
writeFileSync(args.outputPath, JSON.stringify(out));
|
|
135
|
-
const sizeMB = (Buffer.byteLength(JSON.stringify(out)) / 1024 / 1024).toFixed(1);
|
|
136
|
-
console.error(`Wrote ${args.outputPath} (${sizeMB} MB) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`);
|
|
137
|
-
}
|
|
138
|
-
//# sourceMappingURL=corpus-stats.js.map
|