@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
package/out/src/align.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAErE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"align.js","sourceRoot":"","sources":["../../src/align.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAGH,OAAO,EAAE,QAAQ,IAAI,mBAAmB,EAAE,MAAM,qBAAqB,CAAA;AAErE,OAAO,EAAE,mBAAmB,EAAkC,MAAM,eAAe,CAAA;AA2CnF;;GAEG;AACH,MAAM,UAAU,QAAQ,CAAC,GAAiB,EAAE,OAAqB,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,mBAAmB,EAAE,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,CAAC,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,IAAI,CAAA;IAEpD,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,OAAO,EAAE,IAAI,EAAE,aAAa,EAAE,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,CAAA;IAClE,CAAC;IAED,mGAAmG;IACnG,oGAAoG;IACpG,+FAA+F;IAC/F,qGAAqG;IACrG,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IACpC,MAAM,UAAU,GAAG,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAA;IAExC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3C,MAAM,CAAC,GAAG,UAAU,CAAC,GAA8B,CAAC,CAAA;QAEpD,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3B,UAAU,CAAC,GAA8B,CAAC,GAAG,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;QAChE,CAAC;IACF,CAAC;IAED,MAAM,cAAc,GAAoB,EAAE,CAAA;IAC1C,MAAM,OAAO,GAA4B,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAG,eAAe,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,GAAG,CAAA;IAE1D,4FAA4F;IAC5F,6FAA6F;IAC7F,8FAA8F;IAC9F,4BAA4B;IAC5B,MAAM,OAAO,GAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAA+C,CAAC,QAAQ,CACjG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,CACnD,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,MAAM,MAAM,GAAG,eAAe,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,KAAK,CAAA;QAC5D,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAA;QAE5E,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,uBAAuB,GAAG,EAAE,EAAE;aAClD,CAAA;QACF,CAAC;QAED,+FAA+F;QAC/F,8FAA8F;QAC9F,yFAAyF;QACzF,8FAA8F;QAC9F,4FAA4F;QAC5F,oDAAoD;QACpD,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACvE,OAAO;gBACN,IAAI,EAAE,aAAa;gBACnB,GAAG,EAAE,EAAE,GAAG,EAAE,MAAM,EAAE,sBAAsB,GAAG,EAAE,EAAE;aACjD,CAAA;QACF,CAAC;QAED,cAAc,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAChD,oBAAoB,CAAC,cAAc,EAAE,EAAE,GAAG,GAAG,EAAE,GAAG,EAAE,CAAC,CAAA;IACrD,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAElD,MAAM,OAAO,GAAe;QAC3B,GAAG,GAAG;QACN,GAAG;QACH,UAAU;QACV,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACjC,MAAM;QACN,uFAAuF;QACvF,mFAAmF;QACnF,WAAW,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QAC/C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;QAC3C,SAAS,EAAE,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KAC3C,CAAA;IAED,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,OAAO,EAAE,CAAA;AACzC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CACnC,KAA+B,EAC/B,GAAuD;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAEnB,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACnE,MAAM,IAAI,KAAK,CACd,wCAAwC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAClF,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,wBAAwB,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CACvE,CAAA;QACF,CAAC;QAED,IAAI,CAAC,KAAK,CAAC;YAAE,SAAQ;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,GAAG,CAAC,CAAE,CAAA;QAE1B,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CACd,sCAAsC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAChF,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;QAED,IAAI,CAAC,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACxB,MAAM,IAAI,KAAK,CACd,mCAAmC,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;gBAC7E,GAAG,IAAI,CAAC,GAAG,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,cAAc,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,GAAG,GAAG,CACpF,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,IAMnB;IACA,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,EAAE,GAAG,IAAI,CAAA;IAE3D,IAAI,CAAC,MAAM,CAAC,MAAM;QAAE,OAAO,SAAS,CAAA;IAEpC,+FAA+F;IAC/F,gGAAgG;IAChG,gGAAgG;IAChG,uFAAuF;IACvF,gGAAgG;IAChG,2DAA2D;IAC3D,IAAI,SAAqD,CAAA;IACzD,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAA;QAE1C,IAAI,GAAG,KAAK,CAAC,CAAC;YAAE,MAAK;QACrB,MAAM,GAAG,GAAG,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;QAE/B,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,OAAO,CAAC,EAAE,CAAC;YACzC,IAAI,iBAAiB,CAAC,QAAQ,EAAE,GAAG,EAAE,GAAG,CAAC;gBAAE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;YACrE,SAAS,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;QAClC,CAAC;QAED,IAAI,GAAG,GAAG,GAAG,CAAC,CAAA;IACf,CAAC;IAED,IAAI,SAAS;QAAE,OAAO,SAAS,CAAA;IAE/B,IAAI,eAAe,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IAE1C,sFAAsF;IACtF,sFAAsF;IACtF,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACjD,IAAI,eAAe,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,OAAO,CAAC;YAAE,SAAQ;QAClD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,MAAM;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA,CAAC,+BAA+B;QACxF,MAAM,CAAC,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,IAAI,eAAe;YAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,GAAG,EAAE,CAAA;IAC5D,CAAC;IAED,OAAO,SAAS,CAAA;AACjB,CAAC;AAED,MAAM,SAAS,GAAG,eAAe,CAAA;AAEjC;;GAEG;AACH,SAAS,iBAAiB,CAAC,QAAgB,EAAE,KAAa,EAAE,GAAW;IACtE,MAAM,MAAM,GAAG,KAAK,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC,CAAE,CAAC,CAAA;IACnE,MAAM,KAAK,GAAG,GAAG,KAAK,QAAQ,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAE,CAAC,CAAA;IAExE,OAAO,MAAM,IAAI,KAAK,CAAA;AACvB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa,EAAE,GAAW,EAAE,OAAgC;IACpF,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG;YAAE,OAAO,IAAI,CAAA;IACtC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,MAA4B,EAAE,KAA+B;IACjF,MAAM,GAAG,GAAe,EAAE,CAAA;IAC1B,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAA;IAElC,qBAAqB;IAErB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,QAAQ,GAAa,GAAG,CAAA;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEnB,IAAI,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC,GAAG,EAAE,CAAC;gBAClD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACtB,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;oBACnC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;gBAChB,CAAC;qBAAM,CAAC;oBACP,QAAQ,GAAG,KAAK,CAAC,CAAC,GAAG,EAAc,CAAA;gBACpC,CAAC;gBAED,MAAK;YACN,CAAC;QACF,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC"}
|
package/out/src/build.d.ts
CHANGED
|
@@ -50,13 +50,21 @@ import { type ShardManifest } from "./parquet.ts";
|
|
|
50
50
|
import { type AdapterRunManifest } from "./runner.ts";
|
|
51
51
|
import { type SplitManifest } from "./split.ts";
|
|
52
52
|
import type { AdapterOptions, CorpusAdapter } from "./types.ts";
|
|
53
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Stage tags surfaced to `onProgress`.
|
|
55
|
+
*/
|
|
54
56
|
export type BuildStage = "adapter-run" | "align" | "split" | "shard" | "manifest";
|
|
55
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Per-invocation options for `buildCorpus`.
|
|
59
|
+
*/
|
|
56
60
|
export interface BuildCorpusOptions {
|
|
57
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* Root output directory. All build artifacts land beneath it.
|
|
63
|
+
*/
|
|
58
64
|
outputDir: string;
|
|
59
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* Corpus version (e.g. `"0.1.0"`). Stamped onto every row + into the output dir name.
|
|
67
|
+
*/
|
|
60
68
|
corpusVersion: string;
|
|
61
69
|
/**
|
|
62
70
|
* Adapters to drive, in order. Defaults to `defaultAdapterRegistry.list()`. Pass an explicit list to filter (e.g.
|
|
@@ -68,11 +76,17 @@ export interface BuildCorpusOptions {
|
|
|
68
76
|
* noted in the manifest).
|
|
69
77
|
*/
|
|
70
78
|
adapterInputs: Record<string, AdapterOptions>;
|
|
71
|
-
/**
|
|
79
|
+
/**
|
|
80
|
+
* Enable synthesis pass. Default `true`. Set `false` for fixture-driven smoke tests.
|
|
81
|
+
*/
|
|
72
82
|
synthesize?: boolean;
|
|
73
|
-
/**
|
|
83
|
+
/**
|
|
84
|
+
* Forwarded to `writeShards`. Default 1_000_000.
|
|
85
|
+
*/
|
|
74
86
|
rowsPerShard?: number;
|
|
75
|
-
/**
|
|
87
|
+
/**
|
|
88
|
+
* Progress hook. Errors thrown abort the build.
|
|
89
|
+
*/
|
|
76
90
|
onProgress?: (stage: BuildStage, message: string) => void;
|
|
77
91
|
/**
|
|
78
92
|
* License kinds to PURPOSELY exclude from this build (#26). Compiled patterns (see `compileLicenseExcludes` /
|
|
@@ -82,7 +96,9 @@ export interface BuildCorpusOptions {
|
|
|
82
96
|
*/
|
|
83
97
|
excludeLicenses?: readonly RegExp[];
|
|
84
98
|
}
|
|
85
|
-
/**
|
|
99
|
+
/**
|
|
100
|
+
* Top-level manifest tying every stage together.
|
|
101
|
+
*/
|
|
86
102
|
export interface BuildCorpusManifest {
|
|
87
103
|
corpus_version: string;
|
|
88
104
|
built_at: string;
|
package/out/src/build.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAYH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF
|
|
1
|
+
{"version":3,"file":"build.d.ts","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAYH,OAAO,EAAe,KAAK,aAAa,EAAE,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAc,KAAK,kBAAkB,EAAE,MAAM,aAAa,CAAA;AACjE,OAAO,EAIN,KAAK,aAAa,EAElB,MAAM,YAAY,CAAA;AAEnB,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAc,MAAM,YAAY,CAAA;AAEzF;;GAEG;AACH,MAAM,MAAM,UAAU,GAAG,aAAa,GAAG,OAAO,GAAG,OAAO,GAAG,OAAO,GAAG,UAAU,CAAA;AAEjF;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IAEjB;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,QAAQ,CAAC,EAAE,SAAS,aAAa,EAAE,CAAA;IAEnC;;;OAGG;IACH,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAA;IAE7C;;OAEG;IACH,UAAU,CAAC,EAAE,OAAO,CAAA;IAEpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,KAAK,IAAI,CAAA;IAEzD;;;;;OAKG;IACH,eAAe,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACnC;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,kBAAkB,EAAE,CAAA;IAC9B,gBAAgB,EAAE,MAAM,EAAE,CAAA;IAC1B,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAC,UAAU,CAAC,CAAA;KAAE,CAAA;IAChF,MAAM,EAAE;QAAE,MAAM,EAAE,aAAa,CAAC,QAAQ,CAAC,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAC/D,gBAAgB,EAAE,MAAM,CAAA;IACxB,kBAAkB,EAAE,MAAM,CAAA;IAC1B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAChC,mBAAmB,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;GAMG;AACH,wBAAsB,WAAW,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC,CAyNxF"}
|
package/out/src/build.js
CHANGED
|
@@ -160,12 +160,12 @@ export async function buildCorpus(opts) {
|
|
|
160
160
|
try {
|
|
161
161
|
result = alignRow(r);
|
|
162
162
|
}
|
|
163
|
-
catch (
|
|
163
|
+
catch (error) {
|
|
164
164
|
// Last-resort robustness (2026-06-12): no single row may crash a multi-hour build.
|
|
165
165
|
// alignRow's targeted paths normalize/quarantine known issues with specific reasons;
|
|
166
166
|
// this catches any UNKNOWN throw (e.g. assertSpanInvariants on an unforeseen span
|
|
167
167
|
// shape) → quarantine + continue. A spike in `align-threw` reasons is a finding.
|
|
168
|
-
writeQuarantine(r, `align-threw:${
|
|
168
|
+
writeQuarantine(r, `align-threw:${error.message.slice(0, 160)}`);
|
|
169
169
|
quarantined++;
|
|
170
170
|
continue;
|
|
171
171
|
}
|
|
@@ -213,7 +213,7 @@ export async function buildCorpus(opts) {
|
|
|
213
213
|
});
|
|
214
214
|
// License-set visibility (#26): loudly report the resolved license set so a build is an obvious
|
|
215
215
|
// deliberate act — especially a proprietary-weights build (did you pass --exclude-share-alike?).
|
|
216
|
-
const licenseSummary = [...licenseCounts.entries()].
|
|
216
|
+
const licenseSummary = [...licenseCounts.entries()].toSorted((a, b) => b[1] - a[1]);
|
|
217
217
|
opts.onProgress?.("manifest", `license set: ${licenseSummary.map(([l, c]) => `${l}=${c}`).join(", ")}` +
|
|
218
218
|
(excludedByLicense > 0
|
|
219
219
|
? ` | EXCLUDED ${excludedByLicense} rows by --exclude-licenses`
|
package/out/src/build.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACvF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;
|
|
1
|
+
{"version":3,"file":"build.js","sourceRoot":"","sources":["../../src/build.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+CG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,YAAY,EAAoB,MAAM,SAAS,CAAA;AACvF,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAA;AAC7C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAA;AACrC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AAC9C,OAAO,EAAE,WAAW,EAAsB,MAAM,cAAc,CAAA;AAC9D,OAAO,EAAE,UAAU,EAA2B,MAAM,aAAa,CAAA;AACjE,OAAO,EACN,eAAe,EACf,WAAW,EACX,mCAAmC,GAGnC,MAAM,YAAY,CAAA;AACnB,OAAO,EAAE,8BAA8B,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AA8E/E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAwB;IACzD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,sBAAsB,CAAC,IAAI,EAAE,CAAA;IAC/D,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,IAAI,CAAA;IAC1C,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,QAAQ,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAEzC,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,eAAe,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,cAAc,CAAC,CAAA;IAC5D,MAAM,KAAK,CAAC,eAAe,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEjD,mBAAmB;IACnB,MAAM,WAAW,GAAyB,EAAE,CAAA;IAC5C,MAAM,OAAO,GAAa,EAAE,CAAA;IAE5B,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,cAAc,GAAG,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;QAErD,IAAI,CAAC,cAAc,EAAE,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,wBAAwB,CAAC,CAAA;YAE/E,SAAQ;QACT,CAAC;QAED,gGAAgG;QAChG,2FAA2F;QAC3F,+FAA+F;QAC/F,8FAA8F;QAC9F,sGAAsG;QACtG,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;QACpD,MAAM,cAAc,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;QAExD,IACC,OAAO,CAAC,gBAAgB,KAAK,GAAG;YAChC,UAAU,CAAC,cAAc,CAAC;YAC1B,UAAU,CAAC,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC,EAC9C,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,cAAc,EAAE,MAAM,CAAC,CAAuB,CAAA;YACrF,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,YAAY,MAAM,CAAC,OAAO,kBAAkB,CAAC,CAAA;YACnG,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAExB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,EAAE,WAAW,OAAO,CAAC,EAAE,EAAE,CAAC,CAAA;QAEzD,MAAM,CAAC,GAAG,MAAM,UAAU,CAAC;YAC1B,OAAO;YACP,cAAc;YACd,SAAS,EAAE,eAAe;YAC1B,aAAa,EAAE,IAAI,CAAC,aAAa;SACjC,CAAC,CAAA;QAEF,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACpB,CAAC;IAED,yFAAyF;IACzF,8FAA8F;IAC9F,0FAA0F;IAC1F,0FAA0F;IAC1F,0EAA0E;IAC1E,MAAM,YAAY,GAA8B;QAC/C,KAAK,EAAE,IAAI,CAAC,eAAe,EAAE,qBAAqB,CAAC;QACnD,GAAG,EAAE,IAAI,CAAC,eAAe,EAAE,mBAAmB,CAAC;QAC/C,IAAI,EAAE,IAAI,CAAC,eAAe,EAAE,oBAAoB,CAAC;KACjD,CAAA;IAED,MAAM,cAAc,GAAmC;QACtD,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAClE,GAAG,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;QAC9D,IAAI,EAAE,iBAAiB,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;KAChE,CAAA;IAED,MAAM,cAAc,GAAG,IAAI,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAA;IAChE,MAAM,gBAAgB,GAAG,iBAAiB,CAAC,cAAc,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhF,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,MAAM,QAAQ,GAAG,eAAe,EAAE,CAAA;IAClC,8FAA8F;IAC9F,mGAAmG;IACnG,2EAA2E;IAC3E,MAAM,eAAe,GAAG,IAAI,CAAC,eAAe,IAAI,EAAE,CAAA;IAClD,MAAM,aAAa,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC/C,IAAI,iBAAiB,GAAG,CAAC,CAAA;IAEzB,MAAM,eAAe,GAAG,CAAC,GAAiB,EAAE,MAAc,EAAQ,EAAE;QACnE,gBAAgB,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAA;IAC/D,CAAC,CAAA;IAED,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,YAAY,UAAU,CAAC,UAAU,EAAE,CAAC,CAAA;QAE/D,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,WAAW,CAAe,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1E,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAEzE,0FAA0F;YAC1F,sFAAsF;YACtF,0FAA0F;YAC1F,2FAA2F;YAC3F,IAAI,eAAe,CAAC,GAAG,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,CAAC;gBACnD,iBAAiB,EAAE,CAAA;gBAEnB,SAAQ;YACT,CAAC;YAED,MAAM,MAAM,GAAmB,CAAC,GAAG,CAAC,CAAA;YAEpC,IAAI,UAAU,EAAE,CAAC;gBAChB,KAAK,MAAM,GAAG,IAAI,aAAa,CAAC,GAAG,EAAE,8BAA8B,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,EAAE,CAAC;oBACnF,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;gBACjB,CAAC;YACF,CAAC;YAED,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;gBACxB,IAAI,MAAmC,CAAA;gBAEvC,IAAI,CAAC;oBACJ,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;gBACrB,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBAChB,mFAAmF;oBACnF,qFAAqF;oBACrF,kFAAkF;oBAClF,iFAAiF;oBACjF,eAAe,CAAC,CAAC,EAAE,eAAgB,KAAe,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAA;oBAE3E,WAAW,EAAE,CAAA;oBAEb,SAAQ;gBACT,CAAC;gBAED,IAAI,MAAM,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;oBAC/B,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;oBAC/C,cAAc,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;oBAE9D,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;oBAEf,OAAO,EAAE,CAAA;gBACV,CAAC;qBAAM,CAAC;oBACP,eAAe,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;oBAErC,WAAW,EAAE,CAAA;gBACd,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC;QAC/C,CAAC,CAAC,GAAG,EAAE,CAAA;IACR,CAAC;IAED,gBAAgB,CAAC,GAAG,EAAE,CAAA;IACtB,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAA;IAEjG,sFAAsF;IACtF,uFAAuF;IACvF,yEAAyE;IACzE,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,aAAa,OAAO,eAAe,CAAC,CAAA;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAA;IAEhD,MAAM,WAAW,GAAG,MAAM,mCAAmC,CAAC;QAC7D,YAAY;QACZ,SAAS,EAAE,SAAS;QACpB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,MAAM;QACN,QAAQ;KACR,CAAC,CAAA;IAEF,4FAA4F;IAC5F,0FAA0F;IAC1F,IAAI,CAAC,UAAU,EAAE,CAAC,OAAO,EAAE,wBAAwB,CAAC,CAAA;IAEpD,MAAM,aAAa,GAAG,MAAM,WAAW,CACtC;QACC,KAAK,EAAE,WAAW,CAAa,YAAY,CAAC,KAAK,CAAC;QAClD,GAAG,EAAE,WAAW,CAAa,YAAY,CAAC,GAAG,CAAC;QAC9C,IAAI,EAAE,WAAW,CAAa,YAAY,CAAC,IAAI,CAAC;KAChD,EACD;QACC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,YAAY;KACZ,CACD,CAAA;IAED,gGAAgG;IAChG,iGAAiG;IACjG,MAAM,cAAc,GAAG,CAAC,GAAG,aAAa,CAAC,OAAO,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IAEnF,IAAI,CAAC,UAAU,EAAE,CAChB,UAAU,EACV,gBAAgB,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;QACvE,CAAC,iBAAiB,GAAG,CAAC;YACrB,CAAC,CAAC,eAAe,iBAAiB,6BAA6B;YAC/D,CAAC,CAAC,iDAAiD,CAAC,CACtD,CAAA;IAED,yBAAyB;IACzB,IAAI,CAAC,UAAU,EAAE,CAAC,UAAU,EAAE,iCAAiC,CAAC,CAAA;IAEhE,MAAM,QAAQ,GAAwB;QACrC,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,QAAQ,EAAE,WAAW;QACrB,gBAAgB,EAAE,OAAO;QACzB,MAAM,EAAE,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE;QACzC,MAAM,EAAE,EAAE,MAAM,EAAE,aAAa,CAAC,MAAM,EAAE,UAAU,EAAE,aAAa,CAAC,UAAU,EAAE;QAC9E,gBAAgB,EAAE,WAAW;QAC7B,kBAAkB,EAAE,OAAO;QAC3B,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,cAAc,CAAC;QAC5C,mBAAmB,EAAE,iBAAiB;KACtC,CAAA;IAED,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAExG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,WAAW,CAAI,IAAY;IAC1C,mFAAmF;IACnF,kFAAkF;IAClF,qEAAqE;IACrE,KAAK,CAAC,CAAC,eAAe,CAAC,SAAS,CAAI,IAAI,CAAC,CAAA;AAC1C,CAAC;AAED,SAAS,SAAS,CAAC,CAAc;IAChC,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;QACxB,CAAC,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -12,7 +12,9 @@
|
|
|
12
12
|
* `statefp` columns (e.g. `"50"`) into a `region` component (e.g. `"VT"`) without an extra DB
|
|
13
13
|
* join.
|
|
14
14
|
*/
|
|
15
|
-
/**
|
|
15
|
+
/**
|
|
16
|
+
* Per-state record: two-letter postal abbreviation + full canonical display name.
|
|
17
|
+
*/
|
|
16
18
|
export interface USStateInfo {
|
|
17
19
|
abbreviation: string;
|
|
18
20
|
name: string;
|
|
@@ -22,7 +24,9 @@ export interface USStateInfo {
|
|
|
22
24
|
* (PR, GU, VI, MP, AS). Codes are two-digit zero-padded strings to match TIGER column `statefp`.
|
|
23
25
|
*/
|
|
24
26
|
export declare const US_FIPS_STATE: Readonly<Record<string, USStateInfo>>;
|
|
25
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Lookup helper. Returns null when the FIPS code isn't recognized.
|
|
29
|
+
*/
|
|
26
30
|
export declare function lookupFipsState(statefp: string | null | undefined): USStateInfo | null;
|
|
27
31
|
/**
|
|
28
32
|
* Inverted view: two-letter postal abbreviation → `USStateInfo`. Built once at module load. Used by adapters whose
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"us-fips-state.d.ts","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH
|
|
1
|
+
{"version":3,"file":"us-fips-state.d.ts","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,YAAY,EAAE,MAAM,CAAA;IACpB,IAAI,EAAE,MAAM,CAAA;CACZ;AAED;;;GAGG;AACH,eAAO,MAAM,aAAa,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CA0D9D,CAAA;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAItF;AAED;;;GAGG;AACH,eAAO,MAAM,wBAAwB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAE1E,CAAA;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,WAAW,GAAG,IAAI,CAInG"}
|
|
@@ -75,7 +75,9 @@ export const US_FIPS_STATE = Object.freeze({
|
|
|
75
75
|
"72": { abbreviation: "PR", name: "Puerto Rico" },
|
|
76
76
|
"78": { abbreviation: "VI", name: "Virgin Islands" },
|
|
77
77
|
});
|
|
78
|
-
/**
|
|
78
|
+
/**
|
|
79
|
+
* Lookup helper. Returns null when the FIPS code isn't recognized.
|
|
80
|
+
*/
|
|
79
81
|
export function lookupFipsState(statefp) {
|
|
80
82
|
if (!statefp)
|
|
81
83
|
return null;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"us-fips-state.js","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;
|
|
1
|
+
{"version":3,"file":"us-fips-state.js","sourceRoot":"","sources":["../../../src/codex/us-fips-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAUH;;;GAGG;AACH,MAAM,CAAC,MAAM,aAAa,GAA0C,MAAM,CAAC,MAAM,CAAC;IACjF,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,sBAAsB,EAAE;IAC1D,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,cAAc,EAAE;IAClD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE;IAC3C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,UAAU,EAAE;IAC9C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,YAAY,EAAE;IAChD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,eAAe,EAAE;IACnD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE;IAC/C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE;IAC7C,cAAc;IACd,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACpD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1C,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,0BAA0B,EAAE;IAC9D,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE;IACjD,IAAI,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,IAAI,EAAE,gBAAgB,EAAE;CACpD,CAAC,CAAA;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAAkC;IACjE,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAA;IAEzB,OAAO,aAAa,CAAC,OAAO,CAAC,IAAI,IAAI,CAAA;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAA0C,MAAM,CAAC,MAAM,CAC3F,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,CAAC,CAAC,CACzF,CAAA;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,YAAuC;IAC9E,IAAI,CAAC,YAAY;QAAE,OAAO,IAAI,CAAA;IAE9B,OAAO,wBAAwB,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC,IAAI,IAAI,CAAA;AACpE,CAAC"}
|
package/out/src/golden.d.ts
CHANGED
|
@@ -19,7 +19,9 @@
|
|
|
19
19
|
* creep in over time as new entries land.
|
|
20
20
|
*/
|
|
21
21
|
import { type ComponentTag } from "@mailwoman/core/types";
|
|
22
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* One entry in a golden `.jsonl` file.
|
|
24
|
+
*/
|
|
23
25
|
export interface GoldenEntry {
|
|
24
26
|
raw: string;
|
|
25
27
|
components: Partial<Record<ComponentTag, string>>;
|
|
@@ -27,24 +29,36 @@ export interface GoldenEntry {
|
|
|
27
29
|
source: "golden";
|
|
28
30
|
notes?: string;
|
|
29
31
|
}
|
|
30
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Per-entry validation failure.
|
|
34
|
+
*/
|
|
31
35
|
export interface GoldenIssue {
|
|
32
36
|
file: string;
|
|
33
37
|
line: number;
|
|
34
38
|
reason: string;
|
|
35
39
|
}
|
|
36
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* Aggregate report from `validateGoldenDir`.
|
|
42
|
+
*/
|
|
37
43
|
export interface GoldenReport {
|
|
38
44
|
entries: number;
|
|
39
45
|
files: number;
|
|
40
46
|
issues: GoldenIssue[];
|
|
41
47
|
}
|
|
42
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
|
|
50
|
+
*/
|
|
43
51
|
export declare function parseGoldenLine(line: string): GoldenEntry;
|
|
44
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
|
|
54
|
+
*/
|
|
45
55
|
export declare function unreachableComponents(entry: GoldenEntry): ComponentTag[];
|
|
46
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Validate one `.jsonl` file end-to-end, returning a list of issues.
|
|
58
|
+
*/
|
|
47
59
|
export declare function validateGoldenFile(path: string): Promise<GoldenIssue[]>;
|
|
48
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Validate every `.jsonl` in a golden directory.
|
|
62
|
+
*/
|
|
49
63
|
export declare function validateGoldenDir(dir: string): Promise<GoldenReport>;
|
|
50
64
|
//# sourceMappingURL=golden.d.ts.map
|
package/out/src/golden.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"golden.d.ts","sourceRoot":"","sources":["../../src/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAMzE
|
|
1
|
+
{"version":3,"file":"golden.d.ts","sourceRoot":"","sources":["../../src/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAKH,OAAO,EAAkB,KAAK,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAMzE;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IACjD,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,QAAQ,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,WAAW,EAAE,CAAA;CACrB;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAgCzD;AAED;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,WAAW,GAAG,YAAY,EAAE,CAWxE;AAED;;GAEG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CA2B7E;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAc1E"}
|
package/out/src/golden.js
CHANGED
|
@@ -23,10 +23,12 @@ import { extname, join } from "node:path";
|
|
|
23
23
|
import { COMPONENT_TAGS } from "@mailwoman/core/types";
|
|
24
24
|
import { reconcileComponents } from "./format.js";
|
|
25
25
|
const TAG_SET = new Set(COMPONENT_TAGS);
|
|
26
|
-
/**
|
|
26
|
+
/**
|
|
27
|
+
* Parse a single JSONL line into a `GoldenEntry`. Throws on schema violations.
|
|
28
|
+
*/
|
|
27
29
|
export function parseGoldenLine(line) {
|
|
28
30
|
const obj = JSON.parse(line);
|
|
29
|
-
if (typeof obj.raw !== "string" || obj.raw.length
|
|
31
|
+
if (typeof obj.raw !== "string" || !obj.raw.length) {
|
|
30
32
|
throw new Error("missing/empty raw");
|
|
31
33
|
}
|
|
32
34
|
if (typeof obj.country !== "string" || !/^[A-Z]{2}$/u.test(obj.country)) {
|
|
@@ -39,7 +41,7 @@ export function parseGoldenLine(line) {
|
|
|
39
41
|
for (const [k, v] of Object.entries(components)) {
|
|
40
42
|
if (!TAG_SET.has(k))
|
|
41
43
|
throw new Error(`unknown ComponentTag: ${k}`);
|
|
42
|
-
if (typeof v !== "string" || v.length
|
|
44
|
+
if (typeof v !== "string" || !v.length) {
|
|
43
45
|
throw new Error(`components.${k} must be a non-empty string`);
|
|
44
46
|
}
|
|
45
47
|
}
|
|
@@ -51,7 +53,9 @@ export function parseGoldenLine(line) {
|
|
|
51
53
|
notes: typeof obj.notes === "string" ? obj.notes : undefined,
|
|
52
54
|
};
|
|
53
55
|
}
|
|
54
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Check that every component in `entry` appears in `entry.raw` (reconciliation-equivalent).
|
|
58
|
+
*/
|
|
55
59
|
export function unreachableComponents(entry) {
|
|
56
60
|
const reconciled = reconcileComponents(entry.components, entry.raw);
|
|
57
61
|
const missing = [];
|
|
@@ -62,7 +66,9 @@ export function unreachableComponents(entry) {
|
|
|
62
66
|
}
|
|
63
67
|
return missing;
|
|
64
68
|
}
|
|
65
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Validate one `.jsonl` file end-to-end, returning a list of issues.
|
|
71
|
+
*/
|
|
66
72
|
export async function validateGoldenFile(path) {
|
|
67
73
|
const text = await readFile(path, "utf8");
|
|
68
74
|
const lines = text.split("\n");
|
|
@@ -74,7 +80,7 @@ export async function validateGoldenFile(path) {
|
|
|
74
80
|
try {
|
|
75
81
|
const entry = parseGoldenLine(line);
|
|
76
82
|
const unreachable = unreachableComponents(entry);
|
|
77
|
-
if (unreachable.length
|
|
83
|
+
if (unreachable.length) {
|
|
78
84
|
issues.push({
|
|
79
85
|
file: path,
|
|
80
86
|
line: i + 1,
|
|
@@ -82,15 +88,17 @@ export async function validateGoldenFile(path) {
|
|
|
82
88
|
});
|
|
83
89
|
}
|
|
84
90
|
}
|
|
85
|
-
catch (
|
|
86
|
-
issues.push({ file: path, line: i + 1, reason:
|
|
91
|
+
catch (error) {
|
|
92
|
+
issues.push({ file: path, line: i + 1, reason: error.message });
|
|
87
93
|
}
|
|
88
94
|
}
|
|
89
95
|
return issues;
|
|
90
96
|
}
|
|
91
|
-
/**
|
|
97
|
+
/**
|
|
98
|
+
* Validate every `.jsonl` in a golden directory.
|
|
99
|
+
*/
|
|
92
100
|
export async function validateGoldenDir(dir) {
|
|
93
|
-
const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").
|
|
101
|
+
const files = (await readdir(dir)).filter((n) => extname(n) === ".jsonl").toSorted();
|
|
94
102
|
const issues = [];
|
|
95
103
|
let entries = 0;
|
|
96
104
|
for (const name of files) {
|
package/out/src/golden.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"golden.js","sourceRoot":"","sources":["../../src/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,cAAc,EAAqB,MAAM,uBAAuB,CAAA;AAEzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,cAAmC,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"golden.js","sourceRoot":"","sources":["../../src/golden.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAA;AACpD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,cAAc,EAAqB,MAAM,uBAAuB,CAAA;AAEzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAS,cAAmC,CAAC,CAAA;AA+BpE;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,IAAY;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAmD,CAAA;IAE9E,IAAI,OAAO,GAAG,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAA;IACrC,CAAC;IAED,IAAI,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC3F,CAAC;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC7B,MAAM,IAAI,KAAK,CAAC,gCAAgC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/E,CAAC;IAED,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,UAAU,IAAI,EAAE,CAA4B,CAAA;IAEpE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACjD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAA;QAElE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACxC,MAAM,IAAI,KAAK,CAAC,cAAc,CAAC,6BAA6B,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,UAAU,EAAE,UAAuC;QACnD,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,QAAQ;QAChB,KAAK,EAAE,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;KAC5D,CAAA;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAAkB;IACvD,MAAM,UAAU,GAAG,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;IACnE,MAAM,OAAO,GAAmB,EAAE,CAAA;IAElC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAmB,EAAE,CAAC;QACnE,IAAI,CAAC,CAAC,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACpD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IACzC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IAC9B,MAAM,MAAM,GAAkB,EAAE,CAAA;IAEhC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAA;QAE7B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,IAAI,CAAC;YACJ,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACnC,MAAM,WAAW,GAAG,qBAAqB,CAAC,KAAK,CAAC,CAAA;YAEhD,IAAI,WAAW,CAAC,MAAM,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE,IAAI;oBACV,IAAI,EAAE,CAAC,GAAG,CAAC;oBACX,MAAM,EAAE,oCAAoC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;iBACpE,CAAC,CAAA;YACH,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,EAAG,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CAAC,GAAW;IAClD,MAAM,KAAK,GAAG,CAAC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;IACpF,MAAM,MAAM,GAAkB,EAAE,CAAA;IAChC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;QAChC,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,QAAQ,CAAC,CAAA;QACrD,MAAM,CAAC,IAAI,CAAC,GAAG,UAAU,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAC7C,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAC3D,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC"}
|
package/out/src/license.d.ts
CHANGED
|
@@ -31,6 +31,8 @@ export declare const SHARE_ALIKE_PATTERN: RegExp;
|
|
|
31
31
|
* regex.
|
|
32
32
|
*/
|
|
33
33
|
export declare function compileLicenseExcludes(spec: string): RegExp[];
|
|
34
|
-
/**
|
|
34
|
+
/**
|
|
35
|
+
* True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
|
|
36
|
+
*/
|
|
35
37
|
export declare function licenseExcluded(license: string | undefined, patterns: readonly RegExp[]): boolean;
|
|
36
38
|
//# sourceMappingURL=license.d.ts.map
|
package/out/src/license.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED
|
|
1
|
+
{"version":3,"file":"license.d.ts","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,eAAO,MAAM,mBAAmB,QAAmD,CAAA;AAEnF;;;;;GAKG;AACH,wBAAgB,sBAAsB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAM7D;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,SAAS,EAAE,QAAQ,EAAE,SAAS,MAAM,EAAE,GAAG,OAAO,CAIjG"}
|
package/out/src/license.js
CHANGED
|
@@ -35,9 +35,11 @@ export function compileLicenseExcludes(spec) {
|
|
|
35
35
|
.split(",")
|
|
36
36
|
.map((s) => s.trim())
|
|
37
37
|
.filter(Boolean)
|
|
38
|
-
.map((s) => new RegExp("^" + s.
|
|
38
|
+
.map((s) => new RegExp("^" + s.replaceAll(/[.*+?^${}()|[\]\\]/g, "\\$&"), "i"));
|
|
39
39
|
}
|
|
40
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* True iff `license` matches any of the exclude `patterns` (empty patterns → never excluded).
|
|
42
|
+
*/
|
|
41
43
|
export function licenseExcluded(license, patterns) {
|
|
42
44
|
const l = license ?? "";
|
|
43
45
|
return patterns.some((p) => p.test(l));
|
package/out/src/license.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,
|
|
1
|
+
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AACjF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IAEvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
|
|
@@ -10,8 +10,10 @@
|
|
|
10
10
|
import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs";
|
|
11
11
|
import type { BufferReaderOptions } from "@dsnp/parquetjs/dist/lib/bufferReader.js";
|
|
12
12
|
import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js";
|
|
13
|
-
import {
|
|
14
|
-
/**
|
|
13
|
+
import type { ParquetSchema, ParquetRecordLike } from "./schema.ts";
|
|
14
|
+
/**
|
|
15
|
+
* A typed Parquet reader, wrapping the base Parquet reader.
|
|
16
|
+
*/
|
|
15
17
|
export declare class ParquetReader<T extends ParquetRecordLike> extends BaseParquetReader implements AsyncDisposable {
|
|
16
18
|
schema: ParquetSchema<T>;
|
|
17
19
|
static openFile<T extends ParquetRecordLike>(filePath: string | URL, options?: BufferReaderOptions): Promise<ParquetReader<T>>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAA;AAEnE;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;WAEV,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,QAAQ,EAAE,MAAM,GAAG,GAAG,EACtB,OAAO,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAMN,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB;WAMrF,kBAAkB,CAAC,CAAC,SAAS,iBAAiB,EACnE,cAAc,EAAE,qBAAqB,EACrC,IAAI,CAAC,EAAE,mBAAmB;IAkBX,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAI7D,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|
|
@@ -9,8 +9,9 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import { ParquetReader as BaseParquetReader } from "@dsnp/parquetjs";
|
|
11
11
|
import { ParquetEnvelopeReader } from "@dsnp/parquetjs/dist/lib/reader.js";
|
|
12
|
-
|
|
13
|
-
|
|
12
|
+
/**
|
|
13
|
+
* A typed Parquet reader, wrapping the base Parquet reader.
|
|
14
|
+
*/
|
|
14
15
|
export class ParquetReader extends BaseParquetReader {
|
|
15
16
|
static async openFile(filePath, options) {
|
|
16
17
|
const envelopeReader = await ParquetEnvelopeReader.openFile(filePath.toString(), options);
|
|
@@ -29,9 +30,9 @@ export class ParquetReader extends BaseParquetReader {
|
|
|
29
30
|
const metadata = await envelopeReader.readFooter();
|
|
30
31
|
return new ParquetReader(metadata, envelopeReader, opts);
|
|
31
32
|
}
|
|
32
|
-
catch (
|
|
33
|
+
catch (error) {
|
|
33
34
|
await envelopeReader.close();
|
|
34
|
-
throw
|
|
35
|
+
throw error;
|
|
35
36
|
}
|
|
36
37
|
}
|
|
37
38
|
[Symbol.asyncIterator]() {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;
|
|
1
|
+
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAI1E;;GAEG;AACH,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAGhF,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,QAAsB,EACtB,OAA6B;QAE7B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;QAEzF,OAAO,aAAa,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,UAAU,CAA8B,MAAc,EAAE,OAA6B;QAC1G,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE9E,OAAO,IAAI,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,kBAAkB,CACvC,cAAqC,EACrC,IAA0B;QAE1B,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;YACpB,OAAO,IAAI,aAAa,CAAI,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QACjE,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAEjC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAElD,OAAO,IAAI,aAAa,CAAI,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QAC5D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,cAAc,CAAC,KAAK,EAAE,CAAA;YAC5B,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC;IAEe,CAAC,MAAM,CAAC,aAAa,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,EAAsC,CAAA;IACzE,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC;CACD"}
|
|
@@ -19,19 +19,23 @@ import { ParquetSchema as BaseParquetSchema } from "@dsnp/parquetjs";
|
|
|
19
19
|
import type { createSBBFParams as BloomFilterCreation } from "@dsnp/parquetjs/dist/lib/bloomFilterIO/bloomFilterWriter.js";
|
|
20
20
|
import type { FieldDefinition } from "@dsnp/parquetjs/dist/lib/declare.js";
|
|
21
21
|
import { LRUCache } from "lru-cache";
|
|
22
|
-
/**
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
/**
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
/**
|
|
22
|
+
/**
|
|
23
|
+
* A Parquet record-like object, i.e. a record with string keys and JSON-serializable values.
|
|
24
|
+
*/
|
|
25
|
+
export type ParquetRecordLike = Record<string, unknown | undefined>;
|
|
26
|
+
/**
|
|
27
|
+
* Typed Parquet schema definition.
|
|
28
|
+
*/
|
|
29
|
+
export type ParquetSchemaDefinition<T = ParquetRecordLike> = Record<Extract<keyof T, string>, FieldDefinition>;
|
|
30
|
+
/**
|
|
31
|
+
* Typed Parquet schema.
|
|
32
|
+
*/
|
|
31
33
|
export declare class ParquetSchema<T> extends BaseParquetSchema {
|
|
32
34
|
schema: ParquetSchemaDefinition<T>;
|
|
33
35
|
}
|
|
34
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns.
|
|
38
|
+
*/
|
|
35
39
|
export declare function createBloomFilters<T>(parquetSchemaDef: ParquetSchemaDefinition<T>, columns: Extract<keyof T, string>[]): BloomFilterCreation[];
|
|
36
40
|
export declare class ParquetSchemaDefinitionCache extends LRUCache<ParquetSchemaDefinition<ParquetRecordLike>, ParquetSchema<ParquetRecordLike>> implements Disposable {
|
|
37
41
|
constructor(max?: number);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,gBAAgB,IAAI,mBAAmB,EAAE,MAAM,6DAA6D,CAAA;AAC1H,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qCAAqC,CAAA;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,gBAAgB,IAAI,mBAAmB,EAAE,MAAM,6DAA6D,CAAA;AAC1H,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,qCAAqC,CAAA;AAC1E,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAEpC;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,GAAG,SAAS,CAAC,CAAA;AAEnE;;GAEG;AACH,MAAM,MAAM,uBAAuB,CAAC,CAAC,GAAG,iBAAiB,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,eAAe,CAAC,CAAA;AAE9G;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,CAAE,SAAQ,iBAAiB;IAC9C,MAAM,EAAE,uBAAuB,CAAC,CAAC,CAAC,CAAA;CAC1C;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EACnC,gBAAgB,EAAE,uBAAuB,CAAC,CAAC,CAAC,EAC5C,OAAO,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,yBAanC;AAED,qBAAa,4BACZ,SAAQ,QAAQ,CAAC,uBAAuB,CAAC,iBAAiB,CAAC,EAAE,aAAa,CAAC,iBAAiB,CAAC,CAC7F,YAAW,UAAU;gBAET,GAAG,SAAO;IAIf,kBAAkB,CAAC,CAAC,SAAS,iBAAiB,EAAE,SAAS,EAAE,uBAAuB,CAAC,CAAC,CAAC,GAAG,aAAa,CAAC,CAAC,CAAC;IAYxG,CAAC,MAAM,CAAC,OAAO,CAAC;CAGvB"}
|
|
@@ -17,10 +17,14 @@
|
|
|
17
17
|
*/
|
|
18
18
|
import { ParquetSchema as BaseParquetSchema } from "@dsnp/parquetjs";
|
|
19
19
|
import { LRUCache } from "lru-cache";
|
|
20
|
-
/**
|
|
20
|
+
/**
|
|
21
|
+
* Typed Parquet schema.
|
|
22
|
+
*/
|
|
21
23
|
export class ParquetSchema extends BaseParquetSchema {
|
|
22
24
|
}
|
|
23
|
-
/**
|
|
25
|
+
/**
|
|
26
|
+
* Given a Parquet schema and a list of columns, create a list of Bloom filters for those columns.
|
|
27
|
+
*/
|
|
24
28
|
export function createBloomFilters(parquetSchemaDef, columns) {
|
|
25
29
|
const bloomFilters = [];
|
|
26
30
|
for (const column of columns) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAGpE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAYpC
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAGpE,OAAO,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAA;AAYpC;;GAEG;AACH,MAAM,OAAO,aAAiB,SAAQ,iBAAiB;CAEtD;AAED;;GAEG;AACH,MAAM,UAAU,kBAAkB,CACjC,gBAA4C,EAC5C,OAAmC;IAEnC,MAAM,YAAY,GAA0B,EAAE,CAAA;IAE9C,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC9B,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,uBAAuB,MAAM,8BAA8B,CAAC,CAAA;QAC7E,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,CAAC,CAAA;IAC9B,CAAC;IAED,OAAO,YAAY,CAAA;AACpB,CAAC;AAED,MAAM,OAAO,4BACZ,SAAQ,QAAsF;IAG9F,YAAY,GAAG,GAAG,IAAI;QACrB,KAAK,CAAC,EAAE,GAAG,EAAE,CAAC,CAAA;IACf,CAAC;IAEM,kBAAkB,CAA8B,SAAqC;QAC3F,MAAM,GAAG,GAAG,SAAuD,CAAA;QACnE,IAAI,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAiC,CAAA;QAE1D,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,MAAM,GAAG,IAAI,aAAa,CAAI,SAAS,CAAC,CAAA;YACxC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,MAA0C,CAAC,CAAA;QAC1D,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACtB,IAAI,CAAC,KAAK,EAAE,CAAA;IACb,CAAC;CACD"}
|
|
@@ -24,13 +24,21 @@ export declare class ParquetWriter<T extends ParquetRecordLike> extends BaseParq
|
|
|
24
24
|
schema: ParquetSchema<T>;
|
|
25
25
|
protected static readonly SchemaDefinitionCache: ParquetSchemaDefinitionCache;
|
|
26
26
|
static openStream<T extends ParquetRecordLike>(schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>, outputStream: WriteStreamMinimal, opts?: WriterOptions): Promise<ParquetWriter<T>>;
|
|
27
|
-
/**
|
|
27
|
+
/**
|
|
28
|
+
* Convenience method to create a new buffered parquet writer that writes to the specified file.
|
|
29
|
+
*/
|
|
28
30
|
static openFile<T extends ParquetRecordLike>(schemaLike: ParquetSchema<T> | ParquetSchemaDefinition<T>, sourcePath: string | Buffer | URL, opts?: WriterOptions): Promise<ParquetWriter<T>>;
|
|
29
|
-
/**
|
|
31
|
+
/**
|
|
32
|
+
* Set a metadata key-value pair on the writer.
|
|
33
|
+
*/
|
|
30
34
|
setMetadata(key: string, value: string): void;
|
|
31
|
-
/**
|
|
35
|
+
/**
|
|
36
|
+
* Append a row to the buffer. If the buffer is full, the data will be written to disk.
|
|
37
|
+
*/
|
|
32
38
|
appendRow(row: T): Promise<void>;
|
|
33
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Flush all buffered data to disk, close the file, and release resources.
|
|
41
|
+
*/
|
|
34
42
|
close(): Promise<void>;
|
|
35
43
|
[Symbol.asyncDispose](): Promise<void>;
|
|
36
44
|
dispose(): Promise<void>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,+BAAqC;WAG9D,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAC3D,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,YAAY,EAAE,kBAAkB,EAChC,IAAI,GAAE,aAAkB,GACtB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB;;GAEG;AACH,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,+BAAqC;WAG9D,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAC3D,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,YAAY,EAAE,kBAAkB,EAChC,IAAI,GAAE,aAAkB,GACtB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B;;OAEG;WACmB,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,GAAG,EACjC,IAAI,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B;;OAEG;IACa,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7D;;OAEG;IACmB,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD;;OAEG;IACmB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|