@mailwoman/corpus 9.1.0 → 9.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/data/PROVENANCE.md +13 -0
- package/data/reviewed-ve-postcode-tuples.json +74 -0
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +4 -3
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +1 -1
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +2 -2
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +1 -1
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/index.d.ts +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -2
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +1 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -5
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -17
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +11 -6
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +2 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +1 -1
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +6 -5
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +10 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -4
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +7 -7
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +9 -9
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +6 -5
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/utils/index.d.ts +7 -1
- package/out/src/adapters/utils/index.d.ts.map +1 -1
- package/out/src/adapters/utils/index.js +10 -2
- package/out/src/adapters/utils/index.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +5 -5
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +4 -15
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +11 -35
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-json-rows.d.ts +32 -0
- package/out/src/adapters/wof-json-rows.d.ts.map +1 -0
- package/out/src/adapters/wof-json-rows.js +45 -0
- package/out/src/adapters/wof-json-rows.js.map +1 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +4 -9
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +12 -37
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/build.d.ts +4 -3
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +7 -4
- package/out/src/build.js.map +1 -1
- package/out/src/index.d.ts +1 -1
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -1
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/runner.d.ts +2 -2
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +1 -1
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +4 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/bare-country.d.ts +25 -0
- package/out/src/shard-recipes/bare-country.d.ts.map +1 -0
- package/out/src/shard-recipes/bare-country.js +87 -0
- package/out/src/shard-recipes/bare-country.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +2 -2
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +6 -5
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +28 -10
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +4 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +72 -11
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +4 -3
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +4 -3
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +3 -3
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.js +1 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +6 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +3 -3
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +5 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-street.js +1 -1
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +2 -2
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +1 -1
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts +47 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.d.ts.map +1 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js +125 -0
- package/out/src/shard-recipes/reviewed-postcode-tail.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +33 -4
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +5 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +2 -2
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.js +3 -3
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.js +2 -2
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +2 -2
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js +1 -1
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts +1 -1
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/sub-venue.js +6 -5
- package/out/src/shard-recipes/sub-venue.js.map +1 -1
- package/out/src/shard-recipes/trailing-region.d.ts +74 -0
- package/out/src/shard-recipes/trailing-region.d.ts.map +1 -0
- package/out/src/shard-recipes/trailing-region.js +157 -0
- package/out/src/shard-recipes/trailing-region.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +2 -2
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts +1 -1
- package/out/src/synthesizers/boundary-stress.d.ts.map +1 -1
- package/out/src/synthesizers/german.d.ts +1 -1
- package/out/src/synthesizers/german.d.ts.map +1 -1
- package/out/src/synthesizers/german.js.map +1 -1
- package/out/src/synthesizers/house-venue.d.ts +1 -1
- package/out/src/synthesizers/house-venue.d.ts.map +1 -1
- package/out/src/synthesizers/house-venue.js +16 -2
- package/out/src/synthesizers/house-venue.js.map +1 -1
- package/out/src/synthesizers/intersection.d.ts +1 -1
- package/out/src/synthesizers/intersection.d.ts.map +1 -1
- package/out/src/synthesizers/no-street.d.ts +1 -1
- package/out/src/synthesizers/no-street.d.ts.map +1 -1
- package/out/src/synthesizers/po-box.d.ts +1 -1
- package/out/src/synthesizers/po-box.d.ts.map +1 -1
- package/out/src/synthesizers/street.d.ts +1 -1
- package/out/src/synthesizers/street.d.ts.map +1 -1
- package/out/src/synthesizers/street.js +3 -2
- package/out/src/synthesizers/street.js.map +1 -1
- package/out/src/synthesizers/utils.d.ts +1 -1
- package/out/src/synthesizers/utils.d.ts.map +1 -1
- package/out/src/synthesizers/utils.js +3 -2
- package/out/src/synthesizers/utils.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +2 -2
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +2 -18
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +12 -3
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +20 -4
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/geonames-dump.d.ts +87 -0
- package/out/src/tools/fetch/geonames-dump.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-dump.js +178 -0
- package/out/src/tools/fetch/geonames-dump.js.map +1 -0
- package/out/src/tools/fetch/geonames-postal.d.ts +76 -0
- package/out/src/tools/fetch/geonames-postal.d.ts.map +1 -0
- package/out/src/tools/fetch/geonames-postal.js +125 -0
- package/out/src/tools/fetch/geonames-postal.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +11 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +11 -0
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +20 -11
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -7
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +18 -17
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -1
- package/out/src/tools/fetch/ourairports.js +7 -2
- package/out/src/tools/fetch/ourairports.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +0 -4
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +1 -1
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +3 -19
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts +4 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +1 -5
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +15 -21
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +34 -25
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-relabel-street.js +2 -2
- package/out/src/tools/golden-relabel-street.js.map +1 -1
- package/out/src/tools/index.d.ts +2 -2
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +2 -2
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +0 -11
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts +4 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +16 -6
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/postcode-triples.d.ts +175 -0
- package/out/src/tools/postcode-triples.d.ts.map +1 -0
- package/out/src/tools/postcode-triples.js +304 -0
- package/out/src/tools/postcode-triples.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -2
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +4 -26
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue/harvest.d.ts +100 -0
- package/out/src/tools/sub-venue/harvest.d.ts.map +1 -0
- package/out/src/tools/sub-venue/harvest.js +168 -0
- package/out/src/tools/sub-venue/harvest.js.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts +50 -0
- package/out/src/tools/sub-venue/head-nouns.d.ts.map +1 -0
- package/out/src/tools/sub-venue/head-nouns.js +210 -0
- package/out/src/tools/sub-venue/head-nouns.js.map +1 -0
- package/out/src/tools/sub-venue/surfaces.d.ts +45 -0
- package/out/src/tools/sub-venue/surfaces.d.ts.map +1 -0
- package/out/src/tools/sub-venue/surfaces.js +77 -0
- package/out/src/tools/sub-venue/surfaces.js.map +1 -0
- package/out/src/tools/sub-venue/table.d.ts +229 -0
- package/out/src/tools/sub-venue/table.d.ts.map +1 -0
- package/out/src/tools/sub-venue/table.js +108 -0
- package/out/src/tools/sub-venue/table.js.map +1 -0
- package/out/src/tools/sub-venue/wikidata.d.ts +26 -0
- package/out/src/tools/sub-venue/wikidata.d.ts.map +1 -0
- package/out/src/tools/sub-venue/wikidata.js +62 -0
- package/out/src/tools/sub-venue/wikidata.js.map +1 -0
- package/out/src/tools/sub-venue-lexicon.d.ts +27 -378
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -1
- package/out/src/tools/sub-venue-lexicon.js +30 -565
- package/out/src/tools/sub-venue-lexicon.js.map +1 -1
- package/out/src/utils/align.d.ts +1 -1
- package/out/src/utils/align.d.ts.map +1 -1
- package/out/src/utils/align.js.map +1 -1
- package/out/src/utils/golden.js +1 -1
- package/out/src/utils/golden.js.map +1 -1
- package/out/src/utils/license.js +1 -1
- package/out/src/utils/license.js.map +1 -1
- package/out/src/utils/parquet.d.ts +24 -5
- package/out/src/utils/parquet.d.ts.map +1 -1
- package/out/src/utils/parquet.js +6 -2
- package/out/src/utils/parquet.js.map +1 -1
- package/out/src/utils/split.d.ts +1 -1
- package/out/src/utils/split.d.ts.map +1 -1
- package/out/src/utils/split.js +2 -2
- package/out/src/utils/split.js.map +1 -1
- package/package.json +230 -10
- package/src/adapters/ban/adapter.ts +6 -4
- package/src/adapters/fcc-bdc/adapter.ts +4 -3
- package/src/adapters/geonames/adapter.ts +3 -2
- package/src/adapters/geonames-postal/adapter.ts +4 -3
- package/src/adapters/gnaf/adapter.ts +3 -2
- package/src/adapters/index.ts +2 -2
- package/src/adapters/openaddresses/adapter.ts +4 -3
- package/src/adapters/overture/adapter.ts +3 -2
- package/src/adapters/state-hi-schools/adapter.ts +8 -6
- package/src/adapters/state-ia-contractors/adapter.ts +13 -7
- package/src/adapters/state-ny-notaries/adapter.ts +13 -31
- package/src/adapters/state-tx-notaries/adapter.ts +13 -7
- package/src/adapters/synth-po-box/adapter.ts +5 -4
- package/src/adapters/tiger/adapter.ts +3 -2
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +8 -6
- package/src/adapters/usgov-imls-pls/adapter.ts +12 -7
- package/src/adapters/usgov-irs-bmf/adapter.ts +7 -5
- package/src/adapters/usgov-nad/adapter.ts +9 -8
- package/src/adapters/usgov-nppes/adapter.ts +13 -11
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +8 -6
- package/src/adapters/utils/index.ts +16 -3
- package/src/adapters/wof-admin-jp/adapter.ts +9 -12
- package/src/adapters/wof-admin-json/adapter.ts +16 -54
- package/src/adapters/wof-json-rows.ts +79 -0
- package/src/adapters/wof-postalcode-json/adapter.ts +17 -50
- package/src/build.ts +10 -10
- package/src/index.ts +1 -1
- package/src/parquet-wrapper/reader.ts +9 -0
- package/src/runner.ts +2 -7
- package/src/shard-recipes/anchor-absorption.ts +5 -5
- package/src/shard-recipes/bare-country.ts +95 -0
- package/src/shard-recipes/boundary-stress.ts +2 -5
- package/src/shard-recipes/country-balanced.ts +8 -6
- package/src/shard-recipes/cz-pcfirst-preposition.ts +29 -10
- package/src/shard-recipes/fr-admin-split.ts +6 -4
- package/src/shard-recipes/fr-bare-street.ts +85 -10
- package/src/shard-recipes/fr-lieudit.ts +6 -4
- package/src/shard-recipes/fr-order.ts +6 -4
- package/src/shard-recipes/german.ts +4 -3
- package/src/shard-recipes/house-venue.ts +1 -1
- package/src/shard-recipes/index.ts +6 -0
- package/src/shard-recipes/intersection.ts +5 -4
- package/src/shard-recipes/locale.ts +6 -8
- package/src/shard-recipes/no-street.ts +1 -1
- package/src/shard-recipes/po-box-cedex.ts +4 -3
- package/src/shard-recipes/po-box.ts +1 -5
- package/src/shard-recipes/reviewed-postcode-tail.ts +189 -0
- package/src/shard-recipes/scaffold.ts +38 -7
- package/src/shard-recipes/street-affix.ts +4 -3
- package/src/shard-recipes/street-bare.ts +3 -3
- package/src/shard-recipes/street.ts +2 -2
- package/src/shard-recipes/sub-venue-sources.ts +4 -14
- package/src/shard-recipes/sub-venue.ts +10 -8
- package/src/shard-recipes/trailing-region.ts +183 -0
- package/src/shard-recipes/unit.ts +4 -3
- package/src/synthesizers/boundary-stress.ts +1 -1
- package/src/synthesizers/german.ts +2 -1
- package/src/synthesizers/house-venue.ts +17 -3
- package/src/synthesizers/intersection.ts +1 -1
- package/src/synthesizers/no-street.ts +1 -1
- package/src/synthesizers/po-box.ts +1 -1
- package/src/synthesizers/street.ts +5 -3
- package/src/synthesizers/utils.ts +5 -3
- package/src/tools/align-shard.ts +3 -2
- package/src/tools/fetch/ban.ts +2 -22
- package/src/tools/fetch/download.ts +22 -4
- package/src/tools/fetch/geonames-dump.ts +267 -0
- package/src/tools/fetch/geonames-postal.ts +180 -0
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +11 -0
- package/src/tools/fetch/nad.ts +21 -10
- package/src/tools/fetch/nppes.ts +8 -6
- package/src/tools/fetch/openaddresses.ts +18 -22
- package/src/tools/fetch/ourairports.ts +7 -2
- package/src/tools/fetch/ppd.ts +1 -1
- package/src/tools/fetch/state-hi-schools.ts +3 -23
- package/src/tools/fetch/state-sources.ts +1 -1
- package/src/tools/fetch/tiger-full.ts +17 -24
- package/src/tools/golden-expand.ts +44 -33
- package/src/tools/golden-relabel-street.ts +2 -2
- package/src/tools/index.ts +2 -2
- package/src/tools/ingest-csv.ts +0 -13
- package/src/tools/overlay-manifest.ts +20 -8
- package/src/tools/postcode-triples.ts +375 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +10 -35
- package/src/tools/sub-venue/harvest.ts +235 -0
- package/src/tools/sub-venue/head-nouns.ts +242 -0
- package/src/tools/sub-venue/surfaces.ts +95 -0
- package/src/tools/sub-venue/table.ts +281 -0
- package/src/tools/sub-venue/wikidata.ts +85 -0
- package/src/tools/sub-venue-lexicon.ts +67 -886
- package/src/utils/align.ts +2 -1
- package/src/utils/golden.ts +1 -1
- package/src/utils/license.ts +1 -1
- package/src/utils/parquet.ts +22 -8
- package/src/utils/split.ts +4 -3
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"wof-json-rows.js","sourceRoot":"","sources":["../../../src/adapters/wof-json-rows.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAA;AAqBzE;;GAEG;AACH,MAAM,SAAS,CAAC,CAAC,eAAe,CAAC,OAA+B;IAC/D,MAAM,EACL,OAAO,EACP,QAAQ,EACR,cAAc,EACd,SAAS,EACT,eAAe,EACf,UAAU,GAAG,GAAG,EAAE,CAAC,IAAI,EACvB,YAAY,EACZ,WAAW,GACX,GAAG,OAAO,CAAA;IAEX,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,EAAE,IAAI,CAAC,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO;YAAE,OAAM;QAC1C,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,CAAE,CAAA;QAE/B,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;YAAE,SAAQ;QAEjC,KAAK,MAAM,IAAI,IAAI,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC;YACzC,KAAK,MAAM,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBAC/E,IAAI,cAAc,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,IAAI,cAAc,CAAC,KAAK;oBAAE,OAAM;gBAEjF,MAAM,GAAG,GAAG,aAAa,CAAC,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;gBAElF,IAAI,CAAC,GAAG;oBAAE,SAAQ;gBAClB,MAAM,UAAU,GAAG,mBAAmB,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAA;gBAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM;oBAAE,SAAQ;gBAE7C,MAAM;oBACL,GAAG;oBACH,UAAU;oBACV,OAAO,EAAE,MAAM,CAAC,OAAO;oBACvB,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC;oBACvC,MAAM,EAAE,SAAS;oBACjB,SAAS,EAAE,GAAG,SAAS,IAAI,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,MAAM,EAAE;oBACpE,cAAc,EAAE,EAAE;oBAClB,OAAO,EAAE,SAAS;iBAClB,CAAA;gBAED,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;IACF,CAAC;AACF,CAAC"}
|
|
@@ -36,18 +36,14 @@
|
|
|
36
36
|
*
|
|
37
37
|
* License: CC0.
|
|
38
38
|
*/
|
|
39
|
-
import type {
|
|
40
|
-
import type
|
|
41
|
-
import { type
|
|
42
|
-
interface VariantSpec {
|
|
43
|
-
suffix: string;
|
|
44
|
-
components: Partial<Record<ComponentTag, string>>;
|
|
45
|
-
}
|
|
39
|
+
import type { CorpusAdapter } from "#types";
|
|
40
|
+
import { type WOFRecord } from "#utils";
|
|
41
|
+
import { type WOFVariantSpec } from "../wof-json-rows.ts";
|
|
46
42
|
/**
|
|
47
43
|
* Compute hierarchy variants for a postcode record. `selfName` is the postcode surface form (canonical `wof:name` for
|
|
48
44
|
* the `default` slot, a `name:*` localized variant otherwise).
|
|
49
45
|
*/
|
|
50
|
-
export declare function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string):
|
|
46
|
+
export declare function postcodeVariantsFor(row: WOFRecord, ancestry: WOFRecord[], selfName: string): WOFVariantSpec[];
|
|
51
47
|
/**
|
|
52
48
|
* Build the per-record name-slot list. The `default` slot uses `wof:name` verbatim (postcode digits); subsequent slots
|
|
53
49
|
* come from `name:*` variants dedup'd against the default.
|
|
@@ -66,5 +62,4 @@ export declare function createWOFPostalcodeAdapter(): CorpusAdapter;
|
|
|
66
62
|
* The configured adapter instance registered with the corpus builder.
|
|
67
63
|
*/
|
|
68
64
|
export declare const wofPostalcodeAdapter: CorpusAdapter;
|
|
69
|
-
export {};
|
|
70
65
|
//# sourceMappingURL=adapter.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;
|
|
1
|
+
{"version":3,"file":"adapter.d.ts","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAKH,OAAO,KAAK,EAAgC,aAAa,EAAE,MAAM,QAAQ,CAAA;AACzE,OAAO,EAAsD,KAAK,SAAS,EAAE,MAAM,QAAQ,CAAA;AAE3F,OAAO,EAAmB,KAAK,cAAc,EAAE,MAAM,qBAAqB,CAAA;AA6B1E;;;GAGG;AACH,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,GAAG,cAAc,EAAE,CAqC7G;AAED;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,SAAS,GAAG,KAAK,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,CAWlF;AAED;;;GAGG;AACH,eAAO,MAAM,yBAAyB,mBAAmB,CAAA;AAEzD,wBAAgB,0BAA0B,IAAI,aAAa,CAqC1D;AAED;;GAEG;AACH,eAAO,MAAM,oBAAoB,eAA+B,CAAA"}
|
|
@@ -36,8 +36,8 @@
|
|
|
36
36
|
*
|
|
37
37
|
* License: CC0.
|
|
38
38
|
*/
|
|
39
|
-
import { buildAncestryIndex, normalizeNameKey, walkFeatures } from "
|
|
40
|
-
import {
|
|
39
|
+
import { buildAncestryIndex, normalizeNameKey, walkFeatures } from "#utils";
|
|
40
|
+
import { emitWOFJSONRows } from "../wof-json-rows.js";
|
|
41
41
|
const COUNTRY_DISPLAY_NAME = {
|
|
42
42
|
US: "United States of America",
|
|
43
43
|
FR: "France",
|
|
@@ -140,41 +140,16 @@ export function createWOFPostalcodeAdapter() {
|
|
|
140
140
|
}
|
|
141
141
|
const ancestry = buildAncestryIndex(byID);
|
|
142
142
|
// Pass 2: emit postcode rows only, sorted by id for determinism.
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
for (const slot of slots) {
|
|
154
|
-
const variants = postcodeVariantsFor(rec, chain, slot.value);
|
|
155
|
-
for (const variant of variants) {
|
|
156
|
-
if (opts.limit !== undefined && emitted >= opts.limit)
|
|
157
|
-
return;
|
|
158
|
-
const raw = formatAddress(variant.components, rec.country, { separator: ", " });
|
|
159
|
-
if (!raw)
|
|
160
|
-
continue;
|
|
161
|
-
const aligned = reconcileComponents(variant.components, raw);
|
|
162
|
-
if (!Object.keys(aligned).length)
|
|
163
|
-
continue;
|
|
164
|
-
yield {
|
|
165
|
-
raw,
|
|
166
|
-
components: aligned,
|
|
167
|
-
country: rec.country,
|
|
168
|
-
locale: LOCALE_BY_COUNTRY[rec.country],
|
|
169
|
-
source: WOF_POSTALCODE_ADAPTER_ID,
|
|
170
|
-
source_id: `${WOF_POSTALCODE_ADAPTER_ID}-${rec.id}-${slot.key}-${variant.suffix}`,
|
|
171
|
-
corpus_version: "",
|
|
172
|
-
license: "CC0-1.0",
|
|
173
|
-
};
|
|
174
|
-
emitted++;
|
|
175
|
-
}
|
|
176
|
-
}
|
|
177
|
-
}
|
|
143
|
+
yield* emitWOFJSONRows({
|
|
144
|
+
records: byID,
|
|
145
|
+
ancestry,
|
|
146
|
+
adapterOptions: opts,
|
|
147
|
+
adapterID: WOF_POSTALCODE_ADAPTER_ID,
|
|
148
|
+
localeByCountry: LOCALE_BY_COUNTRY,
|
|
149
|
+
shouldEmit: (record) => placetypeToTag(record.placetype) === "postcode",
|
|
150
|
+
nameSlotsFor,
|
|
151
|
+
variantsFor: postcodeVariantsFor,
|
|
152
|
+
});
|
|
178
153
|
},
|
|
179
154
|
};
|
|
180
155
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;
|
|
1
|
+
{"version":3,"file":"adapter.js","sourceRoot":"","sources":["../../../../src/adapters/wof-postalcode-json/adapter.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAMH,OAAO,EAAE,kBAAkB,EAAE,gBAAgB,EAAE,YAAY,EAAkB,MAAM,QAAQ,CAAA;AAE3F,OAAO,EAAE,eAAe,EAAuB,MAAM,qBAAqB,CAAA;AAE1E,MAAM,oBAAoB,GAA2B;IACpD,EAAE,EAAE,0BAA0B;IAC9B,EAAE,EAAE,QAAQ;CACZ,CAAA;AAED,MAAM,iBAAiB,GAA2B;IACjD,EAAE,EAAE,OAAO;IACX,EAAE,EAAE,OAAO;CACX,CAAA;AAED,SAAS,cAAc,CAAC,SAAwC;IAC/D,QAAQ,SAAS,EAAE,CAAC;QACnB,KAAK,SAAS,CAAC;QACf,KAAK,QAAQ;YACZ,OAAO,SAAS,CAAA;QACjB,KAAK,aAAa,CAAC;QACnB,KAAK,QAAQ;YACZ,OAAO,QAAQ,CAAA;QAChB,KAAK,UAAU;YACd,OAAO,UAAU,CAAA;QAClB,KAAK,YAAY;YAChB,OAAO,UAAU,CAAA;QAClB;YACC,OAAO,SAAS,CAAA;IAClB,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,GAAc,EAAE,QAAqB,EAAE,QAAgB;IAC1F,IAAI,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,UAAU;QAAE,OAAO,EAAE,CAAA;IAE3D,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,UAAU,CAAC,CAAA;IACjF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,QAAQ,CAAC,CAAA;IAC7E,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,SAAS,CAAC,CAAA;IAC/E,MAAM,cAAc,GAAG,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,OAAO,EAAE,IAAI,IAAI,GAAG,CAAC,OAAO,CAAA;IAExF,MAAM,QAAQ,GAAqB,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,EAAE,CAAC,CAAA;IAE3F,IAAI,QAAQ,EAAE,CAAC;QACd,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,eAAe;YACvB,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE;SAC3D,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,EAAE,CAAC;QACxB,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,sBAAsB;YAC9B,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE;SAChF,CAAC,CAAA;IACH,CAAC;IAED,IAAI,QAAQ,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;QACnC,QAAQ,CAAC,IAAI,CAAC;YACb,MAAM,EAAE,8BAA8B;YACtC,UAAU,EAAE;gBACX,QAAQ,EAAE,QAAQ;gBAClB,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,MAAM,EAAE,MAAM,CAAC,IAAI;gBACnB,OAAO,EAAE,cAAc;aACvB;SACD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAc;IAC1C,MAAM,IAAI,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;IACxC,MAAM,KAAK,GAA0C,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,IAAI,EAAE,CAAC,CAAA;IAE1F,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,GAAG,CAAC,YAAY,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC;YAAE,SAAQ;QAC7B,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;QACf,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,gBAAgB,CAAC,MAAM,CAAC,EAAE,KAAK,EAAE,CAAC,CAAA;IACrD,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAA;AAEzD,MAAM,UAAU,0BAA0B;IACzC,OAAO;QACN,EAAE,EAAE,yBAAyB;QAC7B,cAAc,EAAE,SAAS;QACzB,WAAW,EACV,wHAAwH;QAEzH,KAAK,CAAC,CAAC,IAAI,CAAC,IAAoB;YAC/B,uFAAuF;YACvF,sFAAsF;YACtF,sFAAsF;YACtF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAqB,CAAA;YAEzC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC;gBAC/E,IAAI,IAAI,CAAC,MAAM,EAAE,OAAO;oBAAE,OAAM;gBAEhC,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO;oBAAE,SAAQ;gBAE1D,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC;oBAAE,SAAQ;gBAC5C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAA;YACtB,CAAC;YAED,MAAM,QAAQ,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAA;YAEzC,iEAAiE;YACjE,KAAK,CAAC,CAAC,eAAe,CAAC;gBACtB,OAAO,EAAE,IAAI;gBACb,QAAQ;gBACR,cAAc,EAAE,IAAI;gBACpB,SAAS,EAAE,yBAAyB;gBACpC,eAAe,EAAE,iBAAiB;gBAClC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,UAAU;gBACvE,YAAY;gBACZ,WAAW,EAAE,mBAAmB;aAChC,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,0BAA0B,EAAE,CAAA"}
|
package/out/src/build.d.ts
CHANGED
|
@@ -46,9 +46,10 @@
|
|
|
46
46
|
* can `rm -rf intermediate/` after the build if disk is tight; the final `corpus-v<version>/` is
|
|
47
47
|
* self-contained.
|
|
48
48
|
*/
|
|
49
|
-
import { type AdapterRunManifest } from "
|
|
50
|
-
import type { AdapterOptions, CorpusAdapter } from "
|
|
51
|
-
import { type ShardManifest
|
|
49
|
+
import { type AdapterRunManifest } from "#runner";
|
|
50
|
+
import type { AdapterOptions, CorpusAdapter } from "#types";
|
|
51
|
+
import { type ShardManifest } from "#utils/parquet";
|
|
52
|
+
import { type SplitManifest } from "#utils/split";
|
|
52
53
|
/**
|
|
53
54
|
* Stage tags surfaced to `onProgress`.
|
|
54
55
|
*/
|
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAWH,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAE7D,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,QAAQ,CAAA;AAGrF,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,gBAAgB,CAAA;AAChE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,cAAc,CAAA;AAErB;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyNxF"}
|
package/out/src/build.js
CHANGED
|
@@ -51,11 +51,14 @@ import { mkdir, writeFile } from "node:fs/promises";
|
|
|
51
51
|
import { join } from "node:path";
|
|
52
52
|
import { $public } from "@mailwoman/core/env";
|
|
53
53
|
import { parseJSONStrict } from "@mailwoman/core/objects";
|
|
54
|
-
import { defaultAdapterRegistry } from "@mailwoman/corpus/adapters/utils";
|
|
55
|
-
import { runAdapter } from "@mailwoman/corpus/runner";
|
|
56
|
-
import { defaultAugmentationsForCountry, synthesizeRow } from "@mailwoman/corpus/synthesizers/utils";
|
|
57
|
-
import { alignRow, licenseExcluded, writeShards, defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "@mailwoman/corpus/utils";
|
|
58
54
|
import { JSONSpliterator } from "spliterator";
|
|
55
|
+
import { defaultAdapterRegistry } from "#adapters/utils";
|
|
56
|
+
import { runAdapter } from "#runner";
|
|
57
|
+
import { defaultAugmentationsForCountry, synthesizeRow } from "#synthesizers/utils";
|
|
58
|
+
import { alignRow } from "#utils/align";
|
|
59
|
+
import { licenseExcluded } from "#utils/license";
|
|
60
|
+
import { writeShards } from "#utils/parquet";
|
|
61
|
+
import { defaultHoldouts, splitForRow, writeSplitManifestsFromLabeledFiles, } from "#utils/split";
|
|
59
62
|
/**
|
|
60
63
|
* Drive the full corpus build to completion.
|
|
61
64
|
*
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACvF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,sBAAsB,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACvF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAA;AACxD,OAAO,EAAE,UAAU,EAA2B,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAA;AAEnF,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AACvC,OAAO,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAE,WAAW,EAAsB,MAAM,gBAAgB,CAAA;AAChE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,cAAc,CAAA;AA6ErB;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAE/E,SAAQ;QACT,CAAC;QAED,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAExD,IACC,OAAO,CAAC,gBAAgB,KAAK,GAAG;YAChC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAqB,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAC,CAAA;YACxF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAExB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QAEzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QAEF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IAED,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBAEnB,SAAQ;YACT,CAAC;YAED,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBAEvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,KAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBAE3E,WAAW,EAAE,CAAA;oBAEb,SAAQ;gBACT,CAAC;gBAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAE9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBAEf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAErC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,CAAC;IAED,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAEhD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IAEpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAExG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,mFAAmF;IACnF,kFAAkF;IAClF,qEAAqE;IACrE,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAI,IAAI,CAAC,CAAA;AAC1C,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
package/out/src/index.d.ts
CHANGED
package/out/src/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,QAAQ,CAAA"}
|
package/out/src/index.js
CHANGED
package/out/src/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,cAAc,2BAA2B,CAAA;AACzC,cAAc,6BAA6B,CAAA;AAC3C,cAAc,qBAAqB,CAAA;AACnC,cAAc,YAAY,CAAA;AAC1B,cAAc,aAAa,CAAA;AAC3B,cAAc,4BAA4B,CAAA;AAC1C,cAAc,0BAA0B,CAAA;AACxC,cAAc,QAAQ,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAEnE;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;WAEV,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,QAAQ,EAAE,MAAM,GAAG,GAAG,EACtB,OAAO,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAMN,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB;WAMrF,kBAAkB,CAAC,CAAC,SAAS,iBAAiB,EACnE,cAAc,EAAE,qBAAqB,EACrC,IAAI,CAAC,EAAE,mBAAmB;IAkBX,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAI1E;;;;;;OAMG;IACW,OAAO,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAEnE;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;WAEV,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,QAAQ,EAAE,MAAM,GAAG,GAAG,EACtB,OAAO,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAMN,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB;WAMrF,kBAAkB,CAAC,CAAC,SAAS,iBAAiB,EACnE,cAAc,EAAE,qBAAqB,EACrC,IAAI,CAAC,EAAE,mBAAmB;IAkBX,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAI1E;;;;;;OAMG;IACW,OAAO,CAAC,CAAC,SAAS,MAAM,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC,EAAE,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAqBvF,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|
|
@@ -46,6 +46,11 @@ export class ParquetReader extends BaseParquetReader {
|
|
|
46
46
|
* caller wanting the whole record should use.
|
|
47
47
|
*/
|
|
48
48
|
async *project(...columns) {
|
|
49
|
+
const missing = columns.filter((column) => !Object.hasOwn(this.schema.schema, column));
|
|
50
|
+
if (missing.length) {
|
|
51
|
+
throw new Error(`Parquet projection requested column${missing.length === 1 ? "" : "s"} absent from the file schema: ` +
|
|
52
|
+
missing.map(String).join(", "));
|
|
53
|
+
}
|
|
49
54
|
const cursor = this.getCursor(columns.map((column) => [column]));
|
|
50
55
|
for (;;) {
|
|
51
56
|
const row = (await cursor.next());
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAI1E;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAGhF,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,QAAsB,EACtB,OAA6B;QAE7B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;QAEzF,OAAO,aAAa,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,UAAU,CAA8B,MAAc,EAAE,OAA6B;QAC1G,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE9E,OAAO,IAAI,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,kBAAkB,CACvC,cAAqC,EACrC,IAA0B;QAE1B,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;YACpB,OAAO,IAAI,aAAa,CAAI,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QACjE,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAEjC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAElD,OAAO,IAAI,aAAa,CAAI,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,cAAc,CAAC,KAAK,EAAE,CAAA;YAC5B,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAEe,CAAC,MAAM,CAAC,aAAa,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,EAAsC,CAAA;IACzE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,CAAC,OAAO,CAAoB,GAAG,OAAY;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAgB,CAAC,CAAC,CAAC,CAAA;QAE1E,SAAS,CAAC;YACT,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAsB,CAAA;YAEtD,IAAI,CAAC,GAAG;gBAAE,OAAM;YAEhB,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC;CACD"}
|
|
1
|
+
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAI1E;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAGhF,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,QAAsB,EACtB,OAA6B;QAE7B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;QAEzF,OAAO,aAAa,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,UAAU,CAA8B,MAAc,EAAE,OAA6B;QAC1G,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE9E,OAAO,IAAI,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,kBAAkB,CACvC,cAAqC,EACrC,IAA0B;QAE1B,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;YACpB,OAAO,IAAI,aAAa,CAAI,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QACjE,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAEjC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAElD,OAAO,IAAI,aAAa,CAAI,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,cAAc,CAAC,KAAK,EAAE,CAAA;YAC5B,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAEe,CAAC,MAAM,CAAC,aAAa,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,EAAsC,CAAA;IACzE,CAAC;IAED;;;;;;OAMG;IACI,KAAK,CAAC,CAAC,OAAO,CAAoB,GAAG,OAAY;QACvD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;QAEtF,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CACd,sCAAsC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,gCAAgC;gBACpG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAC/B,CAAA;QACF,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAgB,CAAC,CAAC,CAAC,CAAA;QAE1E,SAAS,CAAC;YACT,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAsB,CAAA;YAEtD,IAAI,CAAC,GAAG;gBAAE,OAAM;YAEhB,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC;CACD"}
|
package/out/src/runner.d.ts
CHANGED
|
@@ -26,8 +26,8 @@
|
|
|
26
26
|
* The runner does NOT perform alignment, tokenization, synthesis, or sharding into Parquet. Those
|
|
27
27
|
* steps run later, consuming the JSONL shards this writes.
|
|
28
28
|
*/
|
|
29
|
-
import { type AdapterRegistry } from "
|
|
30
|
-
import type { AdapterOptions, CorpusAdapter } from "
|
|
29
|
+
import { type AdapterRegistry } from "#adapters/utils";
|
|
30
|
+
import type { AdapterOptions, CorpusAdapter } from "#types";
|
|
31
31
|
/**
|
|
32
32
|
* Snapshot of the runner's state, emitted on every progress tick.
|
|
33
33
|
*/
|
package/out/src/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,EAAsC,KAAK,eAAe,EAAwB,MAAM,iBAAiB,CAAA;AAChH,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAE,MAAM,QAAQ,CAAA;AAEzE;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,OAAO,EAAE,MAAM,CAAA;IAEf;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IAEb;;OAEG;IACH,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC;;OAEG;IACH,OAAO,EAAE,aAAa,CAAA;IAEtB;;OAEG;IACH,cAAc,EAAE,cAAc,CAAA;IAE9B;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAA;IAE/C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CA2GrF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,GAAG;IAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,cAAc,CAAA;CAAE,GACvG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAgB/B;AAgDD;;GAEG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE"}
|
package/out/src/runner.js
CHANGED
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
import { createWriteStream } from "node:fs";
|
|
30
30
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
31
31
|
import { dirname, join } from "node:path";
|
|
32
|
-
import { canonicalDedupKey, streamingSha256
|
|
32
|
+
import { canonicalDedupKey, streamingSha256 } from "#adapters/utils";
|
|
33
33
|
/**
|
|
34
34
|
* Drive a single adapter to completion.
|
|
35
35
|
*
|
package/out/src/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAA8C,MAAM,iBAAiB,CAAA;AAyFhH;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACvD,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI,CAAA;IAClE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAA;IAEhD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAoB,eAAe,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,cAAc,GAAG,UAAU,CAAA;IACjC,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,YAAY,GAAG,GAAS,EAAE;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO;YACP,OAAO;YACP,KAAK;YACL,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;SAClC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,YAAY,CAAC,+BAA+B,EAAE,YAAY,CAAC,CAAA;YACtE,CAAC;YAED,OAAO,EAAE,CAAA;YACT,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAiB,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,CAAA;YACvE,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;YAEtC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;wBACnC,YAAY,EAAE,CAAA;oBACf,CAAC;oBAED,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;oBACjC,cAAc,GAAG,IAAI,CAAA;oBAErB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+BAA+B,cAAc,CAAC,cAAc,EAAE,wCAAwC,CACtG,CAAA;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;YACF,CAAC;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YAExC,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC,EAAE,CAAC;gBACnC,YAAY,EAAE,CAAA;YACf,CAAC;QACF,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACzC,YAAY,EAAE,CAAA;IAEd,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,OAAO,CAAC,EAAE;QACtB,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,OAAO,CAAC,cAAc;QACvC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO;QACP,OAAO;QACP,OAAO,EAAE,OAAO,GAAG,OAAO;QAC1B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;QAC/B,UAAU;KACV,CAAA;IAED,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE/E,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAyB,EACzB,MAAyG;IAEzG,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,cAAc,CAAA;QAEnF,GAAG,CAAC,IAAI,CACP,MAAM,UAAU,CAAC;YAChB,GAAG,MAAM;YACT,OAAO;YACP,cAAc;SACd,CAAC,CACF,CAAA;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAsB,EAAE,GAAiB;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,oCAAoC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,OAAoB,EAAE,KAAwB;IAC3D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,GAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;QAED,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAA;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anchor-absorption.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,
|
|
1
|
+
{"version":3,"file":"anchor-absorption.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAUH,OAAO,EAA0B,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAwBxE;;;GAGG;AACH,eAAO,MAAM,sBAAsB,EAAE,WAoDpC,CAAA"}
|
|
@@ -14,10 +14,9 @@
|
|
|
14
14
|
import { readFileSync } from "node:fs";
|
|
15
15
|
import { parseJSONStrict } from "@mailwoman/core/objects";
|
|
16
16
|
import { dataRootPath } from "@mailwoman/core/utils";
|
|
17
|
-
import {
|
|
18
|
-
import {
|
|
19
|
-
import {
|
|
20
|
-
import { makeLcg } from "./scaffold.js";
|
|
17
|
+
import { synthesizeAnchorAbsorptionRow } from "#synthesizers/anchor-absorption";
|
|
18
|
+
import { alignRow } from "#utils";
|
|
19
|
+
import { makeLcg, shardSourceID } from "./scaffold.js";
|
|
21
20
|
/**
|
|
22
21
|
* The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through the data-root
|
|
23
22
|
* helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
|
|
@@ -64,7 +63,7 @@ export const anchorAbsorptionRecipe = {
|
|
|
64
63
|
country,
|
|
65
64
|
locale: synth.locale,
|
|
66
65
|
source,
|
|
67
|
-
source_id:
|
|
66
|
+
source_id: shardSourceID(source, { v: String(i) }),
|
|
68
67
|
};
|
|
69
68
|
const aligned = alignRow(canonical);
|
|
70
69
|
if (aligned.kind !== "labeled") {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"anchor-absorption.js","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"anchor-absorption.js","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,6BAA6B,EAAE,MAAM,iCAAiC,CAAA;AAC/E,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAoB,MAAM,eAAe,CAAA;AAExE;;;GAGG;AACH,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;AAExE;;GAEG;AACH,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,CAAC,GAAG,eAAe,CAA0B,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAA;IAC9E,MAAM,IAAI,GAAa,EAAE,CAAA;IAEzB,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,IAAK,CAAC,CAAC,CAAC,CAAY,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAgB;IAClD,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,iGAAiG;IAC9G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,yBAAyB,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAA;QAClC,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,CAAC,CAAA;QAE9C,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,oEAAoE,CAAC,CAAA;QAE5G,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,MAAM,UAAU,GAA2B,EAAE,CAAA;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,6BAA6B,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YACjE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA;YAE1C,mCAAmC;YACnC,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,aAAa,CAAC,MAAM,EAAE,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;aAClD,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,WAAW,EAAE,CAAA;gBAEb,SAAQ;YACT,CAAC;YAED,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAC5G,CAAA;YAED,OAAO,EAAE,CAAA;YACT,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnE,CAAC;QAED,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,UAAU,WAAW,eAAe,CAAC,CAAA;QACrE,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;QAExD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IAClD,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
|
|
7
|
+
* country name, so the class is out of the model's training distribution and the tag is a coin
|
|
8
|
+
* flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
|
|
9
|
+
* `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
|
|
10
|
+
* bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
|
|
11
|
+
* tag itself is right.
|
|
12
|
+
*
|
|
13
|
+
* Source is the codex country table — every ISO canonical English name plus the curated surface
|
|
14
|
+
* forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
|
|
15
|
+
* `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
|
|
16
|
+
* English words), and teaching it here would be an unmeasured claim. Case augmentation is the
|
|
17
|
+
* loader's job, as everywhere.
|
|
18
|
+
*/
|
|
19
|
+
import { type ShardRecipe } from "./scaffold.ts";
|
|
20
|
+
/**
|
|
21
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
22
|
+
* and `description` below for the surface form it generates.
|
|
23
|
+
*/
|
|
24
|
+
export declare const bareCountryRecipe: ShardRecipe;
|
|
25
|
+
//# sourceMappingURL=bare-country.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bare-country.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/bare-country.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAIH,OAAO,EAAiC,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAgC9F;;;GAGG;AACH,eAAO,MAAM,iBAAiB,EAAE,WAqC/B,CAAA"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `bare-country` — the country name as the WHOLE query. Addresses almost never consist of one bare
|
|
7
|
+
* country name, so the class is out of the model's training distribution and the tag is a coin
|
|
8
|
+
* flip: measured on the 2026-08-13 panel, France/Germany/United States/New Zealand parsed
|
|
9
|
+
* `country` while Japan/China/Nigeria/Australia/Deutschland parsed `locality`. The retrieval-side
|
|
10
|
+
* bare-country race (#1651) covers the answer either way; this shard closes the PARSE half so the
|
|
11
|
+
* tag itself is right.
|
|
12
|
+
*
|
|
13
|
+
* Source is the codex country table — every ISO canonical English name plus the curated surface
|
|
14
|
+
* forms (endonyms and long-form names). The 2–3 letter CODES are deliberately excluded: a bare
|
|
15
|
+
* `JP` or `GER` is an abbreviation register with its own ambiguity surface (`IN`, `DE`, `TO` are
|
|
16
|
+
* English words), and teaching it here would be an unmeasured claim. Case augmentation is the
|
|
17
|
+
* loader's job, as everywhere.
|
|
18
|
+
*/
|
|
19
|
+
import { COUNTRY_SURFACE_FORMS, CountryNames, matchCountry } from "@mailwoman/codex/country";
|
|
20
|
+
import { alignAndWrite, makeMulberry32, shardSourceID } from "./scaffold.js";
|
|
21
|
+
/**
|
|
22
|
+
* Surfaces shorter than this are the code register (`JP`, `GER`), not a name — excluded (see the module doc).
|
|
23
|
+
*/
|
|
24
|
+
const MIN_NAME_LENGTH = 4;
|
|
25
|
+
/**
|
|
26
|
+
* Every (surface, iso2) pair the shard emits: the ISO canonical names plus each country's curated surface forms,
|
|
27
|
+
* deduplicated on the surface string (a form shared across countries — none known — would keep its first bearer).
|
|
28
|
+
*/
|
|
29
|
+
function* bareCountrySurfaces() {
|
|
30
|
+
const seen = new Set();
|
|
31
|
+
for (const [iso2, forms] of Object.entries(COUNTRY_SURFACE_FORMS)) {
|
|
32
|
+
for (const surface of forms) {
|
|
33
|
+
if (surface.length < MIN_NAME_LENGTH || seen.has(surface))
|
|
34
|
+
continue;
|
|
35
|
+
seen.add(surface);
|
|
36
|
+
yield { surface, iso2 };
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
for (const name of CountryNames) {
|
|
40
|
+
if (name.length < MIN_NAME_LENGTH || seen.has(name))
|
|
41
|
+
continue;
|
|
42
|
+
const matched = matchCountry(name);
|
|
43
|
+
if (!matched)
|
|
44
|
+
continue;
|
|
45
|
+
seen.add(name);
|
|
46
|
+
yield { surface: name, iso2: matched.iso2 };
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Shard recipe registered with the corpus builder — see the file header for the parse behaviour it exists to exercise,
|
|
51
|
+
* and `description` below for the surface form it generates.
|
|
52
|
+
*/
|
|
53
|
+
export const bareCountryRecipe = {
|
|
54
|
+
name: "bare-country",
|
|
55
|
+
description: "The country name as the whole query (#1651 parse half): ISO names + curated endonyms, no codes",
|
|
56
|
+
mode: "generate",
|
|
57
|
+
async run(opts, write) {
|
|
58
|
+
// Seeded for parity with the other recipes; the surfaces themselves drive the content.
|
|
59
|
+
makeMulberry32(opts.seed);
|
|
60
|
+
let read = 0;
|
|
61
|
+
let emitted = 0;
|
|
62
|
+
let skipped = 0;
|
|
63
|
+
for (const { surface, iso2 } of bareCountrySurfaces()) {
|
|
64
|
+
read++;
|
|
65
|
+
const components = { country: surface };
|
|
66
|
+
const source_id = shardSourceID("synth-bare-country", { country: surface, cc: iso2, v: String(read) });
|
|
67
|
+
const canonical = {
|
|
68
|
+
raw: surface,
|
|
69
|
+
components,
|
|
70
|
+
country: iso2,
|
|
71
|
+
locale: "und",
|
|
72
|
+
source: "synth-bare-country",
|
|
73
|
+
source_id,
|
|
74
|
+
corpus_version: "0.11.0",
|
|
75
|
+
license: "Synthetic — bare-country; surfaces from the codex country table (ISO 3166 + curated forms)",
|
|
76
|
+
};
|
|
77
|
+
if (alignAndWrite(write, canonical, "bare-country")) {
|
|
78
|
+
emitted++;
|
|
79
|
+
}
|
|
80
|
+
else {
|
|
81
|
+
skipped++;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return { read, emitted, skipped };
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
//# sourceMappingURL=bare-country.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"bare-country.js","sourceRoot":"","sources":["../../../src/shard-recipes/bare-country.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,qBAAqB,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,0BAA0B,CAAA;AAE5F,OAAO,EAAE,aAAa,EAAE,cAAc,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9F;;GAEG;AACH,MAAM,eAAe,GAAG,CAAC,CAAA;AAEzB;;;GAGG;AACH,QAAQ,CAAC,CAAC,mBAAmB;IAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,qBAAqB,CAAC,EAAE,CAAC;QACnE,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC7B,IAAI,OAAO,CAAC,MAAM,GAAG,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,SAAQ;YACnE,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;YACjB,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;QACxB,CAAC;IACF,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,IAAI,IAAI,CAAC,MAAM,GAAG,eAAe,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAQ;QAC7D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;QAElC,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QACd,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAA;IAC5C,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAgB;IAC7C,IAAI,EAAE,cAAc;IACpB,WAAW,EAAE,gGAAgG;IAC7G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,uFAAuF;QACvF,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACzB,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,mBAAmB,EAAE,EAAE,CAAC;YACvD,IAAI,EAAE,CAAA;YAEN,MAAM,UAAU,GAA2B,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;YAC/D,MAAM,SAAS,GAAG,aAAa,CAAC,oBAAoB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;YAEtG,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,OAAO;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,oBAAoB;gBAC5B,SAAS;gBACT,cAAc,EAAE,QAAQ;gBACxB,OAAO,EAAE,4FAA4F;aACrG,CAAA;YAED,IAAI,aAAa,CAAC,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,CAAA;YACV,CAAC;iBAAM,CAAC;gBACP,OAAO,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAClC,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;
|
|
1
|
+
{"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAKH,OAAO,EAAkB,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AAkC/E;;;GAGG;AACH,eAAO,MAAM,oBAAoB,EAAE,WA6ClC,CAAA"}
|
|
@@ -11,8 +11,8 @@
|
|
|
11
11
|
*
|
|
12
12
|
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
13
|
*/
|
|
14
|
-
import { synthesizeBoundaryStressRow
|
|
15
|
-
import { alignRow } from "
|
|
14
|
+
import { synthesizeBoundaryStressRow } from "#synthesizers/boundary-stress";
|
|
15
|
+
import { alignRow } from "#utils";
|
|
16
16
|
import { makeMulberry32, shardSourceID } from "./scaffold.js";
|
|
17
17
|
/**
|
|
18
18
|
* Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality drop on bare
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAA+B,2BAA2B,EAAE,MAAM,+BAA+B,CAAA;AACxG,OAAO,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAEjC,OAAO,EAAE,cAAc,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE/E;;;;;;GAMG;AACH,MAAM,OAAO,GAA2C;IACvD,mBAAmB,EAAE,IAAI;IACzB,uBAAuB,EAAE,IAAI;IAC7B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,KAAK;IACnC,2BAA2B,EAAE,KAAK;CAClC,CAAA;AAED,MAAM,GAAG,GAA4C,CAAC,GAAG,EAAE;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,OAAQ,MAAM,CAAC,OAAO,CAAC,OAAO,CAA6C,CAAC,GAAG,CAC9E,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAqC,CAC/D,CAAA;AACF,CAAC,CAAC,EAAE,CAAA;AAEJ,SAAS,YAAY,CAAC,CAAe;IACpC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IAEb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;IAE9C,OAAO,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAChD,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,sFAAsF;QACtF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,MAAM,CAAA;QAClC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,2BAA2B,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAE7F,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YAED,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAA;gBAET,SAAQ;YACT,CAAC;YAED,iGAAiG;YACjG,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAC/G,CAAA;YAED,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;
|
|
1
|
+
{"version":3,"file":"country-balanced.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/country-balanced.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAWH,OAAO,EAAwC,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAgYtF;;;GAGG;AACH,eAAO,MAAM,qBAAqB,EAAE,WAoGnC,CAAA"}
|
|
@@ -24,9 +24,10 @@
|
|
|
24
24
|
* legacy script used.
|
|
25
25
|
*/
|
|
26
26
|
import { COUNTRY_SURFACE_FORMS, CountryNames } from "@mailwoman/codex/country";
|
|
27
|
+
import { isPresent } from "@mailwoman/core/objects";
|
|
27
28
|
import { dataRootPath } from "@mailwoman/core/utils";
|
|
28
|
-
import { stableSourceID } from "
|
|
29
|
-
import { alignRow } from "
|
|
29
|
+
import { stableSourceID } from "#adapters/utils";
|
|
30
|
+
import { alignRow } from "#utils";
|
|
30
31
|
import { makeMulberry32, readZippedCSVRecords } from "./scaffold.js";
|
|
31
32
|
// v2: the country TOKEN is decoupled from the skeleton's locale and drawn from a BROAD pool — every
|
|
32
33
|
// ISO canonical name + every curated surface form (endonyms/abbrevs). Surface forms are over-weighted
|
|
@@ -170,16 +171,16 @@ function renderCountry(random, t, country) {
|
|
|
170
171
|
}
|
|
171
172
|
let body;
|
|
172
173
|
if (order === "us") {
|
|
173
|
-
const regPc = [reg, pc].filter(
|
|
174
|
+
const regPc = [reg, pc].filter(isPresent).join(" ");
|
|
174
175
|
body = `${hn} ${street}, ${loc}${regPc ? ", " + regPc : ""}`;
|
|
175
176
|
}
|
|
176
177
|
else if (order === "fr") {
|
|
177
178
|
// French is NUMBER-street, postcode-city: "84 Route de la Fontaine, 75008 Paris".
|
|
178
|
-
body = `${hn} ${street}, ${[pc, loc].filter(
|
|
179
|
+
body = `${hn} ${street}, ${[pc, loc].filter(isPresent).join(" ")}`;
|
|
179
180
|
}
|
|
180
181
|
else {
|
|
181
182
|
// DE/IT/NL: street-number, postcode-city: "Pariser Platz 1, 10117 Berlin".
|
|
182
|
-
const pcCity = [pc, loc].filter(
|
|
183
|
+
const pcCity = [pc, loc].filter(isPresent).join(" ");
|
|
183
184
|
body = `${street} ${hn}, ${pcCity}`;
|
|
184
185
|
}
|
|
185
186
|
if (!country) {
|