@mailwoman/corpus 8.1.0 → 8.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +7 -1
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +34 -18
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +10 -1
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +43 -19
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -27,9 +27,7 @@
|
|
|
27
27
|
import { existsSync, mkdirSync, writeFileSync } from "node:fs";
|
|
28
28
|
import { basename, dirname, extname, join } from "node:path";
|
|
29
29
|
import { TextSpliterator } from "spliterator";
|
|
30
|
-
|
|
31
|
-
// Core: quote-aware CSV field splitting
|
|
32
|
-
// ---------------------------------------------------------------------------
|
|
30
|
+
//#region Core: quote-aware CSV field splitting
|
|
33
31
|
const COMMA = 44;
|
|
34
32
|
const DOUBLE_QUOTE = 34;
|
|
35
33
|
// These stay hand-rolled on purpose — spliterator ≥ 3.2.0's CSVSpliterator does quote handling
|
|
@@ -58,20 +56,19 @@ function splitCSVLine(line, separator = COMMA) {
|
|
|
58
56
|
function stripQuotes(field) {
|
|
59
57
|
const trimmed = field.trim();
|
|
60
58
|
if (trimmed.length >= 2 && trimmed.startsWith('"') && trimmed.endsWith('"')) {
|
|
61
|
-
return trimmed.slice(1, -1).
|
|
59
|
+
return trimmed.slice(1, -1).replaceAll('""', '"');
|
|
62
60
|
}
|
|
63
61
|
return trimmed;
|
|
64
62
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
// ---------------------------------------------------------------------------
|
|
63
|
+
//#endregion
|
|
64
|
+
//#region Column name normalization
|
|
68
65
|
function normalizeColumnName(raw) {
|
|
69
66
|
return (raw
|
|
70
67
|
.trim()
|
|
71
|
-
.
|
|
68
|
+
.replaceAll(/^"+|"+$/g, "")
|
|
72
69
|
.toLowerCase()
|
|
73
|
-
.
|
|
74
|
-
.
|
|
70
|
+
.replaceAll(/[^a-z0-9]+/g, "_")
|
|
71
|
+
.replaceAll(/^_|_$/g, "") || "unnamed");
|
|
75
72
|
}
|
|
76
73
|
function dedupColumns(names) {
|
|
77
74
|
const seen = new Map();
|
|
@@ -84,7 +81,7 @@ function dedupColumns(names) {
|
|
|
84
81
|
function normalizeField(raw) {
|
|
85
82
|
let s = raw.trim();
|
|
86
83
|
if (s.length >= 2 && s.startsWith('"') && s.endsWith('"')) {
|
|
87
|
-
s = s.slice(1, -1).
|
|
84
|
+
s = s.slice(1, -1).replaceAll('""', '"');
|
|
88
85
|
}
|
|
89
86
|
// Normalize common null-like values
|
|
90
87
|
if (!s || s === "null" || s === "NULL" || s === "N/A" || s === "n/a" || s === "-" || s === "<UNAVAIL>") {
|
|
@@ -214,13 +211,13 @@ async function runIngest(opts) {
|
|
|
214
211
|
}
|
|
215
212
|
db.exec("COMMIT");
|
|
216
213
|
}
|
|
217
|
-
catch (
|
|
214
|
+
catch (error) {
|
|
218
215
|
db.exec("ROLLBACK");
|
|
219
|
-
throw
|
|
216
|
+
throw error;
|
|
220
217
|
}
|
|
221
218
|
};
|
|
222
219
|
const batch = [];
|
|
223
|
-
const BATCH_SIZE =
|
|
220
|
+
const BATCH_SIZE = 10_000;
|
|
224
221
|
for await (const line of TextSpliterator.fromAsync(opts.inputPath)) {
|
|
225
222
|
lineNum++;
|
|
226
223
|
if (lineNum <= opts.skipLines)
|
|
@@ -236,9 +233,9 @@ async function runIngest(opts) {
|
|
|
236
233
|
return null;
|
|
237
234
|
const col = columns[i];
|
|
238
235
|
if (col?.type === "INTEGER" && /^-?\d+$/.test(v))
|
|
239
|
-
return parseInt(v, 10);
|
|
236
|
+
return Number.parseInt(v, 10);
|
|
240
237
|
if (col?.type === "REAL" && /^-?\d+\.?\d+$/.test(v))
|
|
241
|
-
return parseFloat(v);
|
|
238
|
+
return Number.parseFloat(v);
|
|
242
239
|
return v;
|
|
243
240
|
});
|
|
244
241
|
// Pad or truncate to column count
|
|
@@ -251,13 +248,13 @@ async function runIngest(opts) {
|
|
|
251
248
|
doInsert();
|
|
252
249
|
imported += batch.length;
|
|
253
250
|
batch.length = 0;
|
|
254
|
-
if (imported %
|
|
251
|
+
if (imported % 100_000 === 0) {
|
|
255
252
|
process.stderr.write(` ${(imported / 1_000_000).toFixed(1)}M rows...\n`);
|
|
256
253
|
}
|
|
257
254
|
}
|
|
258
255
|
}
|
|
259
256
|
// Flush remaining
|
|
260
|
-
if (batch.length
|
|
257
|
+
if (batch.length) {
|
|
261
258
|
doInsert();
|
|
262
259
|
imported += batch.length;
|
|
263
260
|
}
|
|
@@ -297,7 +294,7 @@ export async function ingestCSV(options) {
|
|
|
297
294
|
const csvName = basename(options.input, extname(options.input));
|
|
298
295
|
await runIngest({
|
|
299
296
|
inputPath: options.input,
|
|
300
|
-
tableName: options.table ?? csvName.
|
|
297
|
+
tableName: options.table ?? csvName.replaceAll(/[^a-zA-Z0-9_]/g, "_"),
|
|
301
298
|
outputPath: options.output ?? join(dirname(options.input), csvName + ".db"),
|
|
302
299
|
sampleSize: options.sample ?? 100,
|
|
303
300
|
separator: options.separator ?? ",",
|
|
@@ -306,4 +303,5 @@ export async function ingestCSV(options) {
|
|
|
306
303
|
dryRun: options.dryRun ?? false,
|
|
307
304
|
});
|
|
308
305
|
}
|
|
306
|
+
//#endregion
|
|
309
307
|
//# sourceMappingURL=ingest-csv.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../../src/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,8EAA8E;AAC9E,wCAAwC;AACxC,8EAA8E;AAE9E,MAAM,KAAK,GAAG,EAAE,CAAA;AAChB,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB,+FAA+F;AAC/F,kGAAkG;AAClG,mGAAmG;AACnG,mGAAmG;AACnG,+FAA+F;AAC/F,oEAAoE;AACpE,SAAS,YAAY,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC5D,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAE7B,IAAI,EAAE,KAAK,YAAY,EAAE,CAAC;YACzB,QAAQ,GAAG,CAAC,QAAQ,CAAA;QACrB,CAAC;aAAM,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;YACjC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IAE9B,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAE5B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAChD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,8EAA8E;AAC9E,4BAA4B;AAC5B,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACpC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IACvC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YACX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAExC,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,iGAAiG;IACjG,qGAAqG;IACrG,qGAAqG;IACrG,sGAAsG;IACtG,8BAA8B;IAC9B,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,2BAA2B;QAC3B,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,UAAU,GAAG,IAAI,CAAA;YACjB,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,UAAU,EAAE,CAAC;QAClC,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5D,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,6BAA6B;IAC7B,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QACF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,uBAAuB;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,QAAQ,eAAe,IAAI,CAAC,SAAS,WAAW,CAAA;IAE3H,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IACpD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAC5C,MAAM,SAAS,GAAG;QACjB,sBAAsB,IAAI,CAAC,SAAS,aAAa,OAAO,IAAI;QAC5D,WAAW;QACX,eAAe,IAAI,CAAC,SAAS,GAAG;QAChC,YAAY,WAAW,YAAY,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,SAAS,SAAS;QACrH,SAAS;QACT,oBAAoB,IAAI,CAAC,SAAS,WAAW;KAC7C,CAAA;IAED,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YACD,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,KAAK,CAAA;IAExB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YACpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QACvD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAExE,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,UAAU,CAAC,CAAC,CAAC,CAAA;YAEzE,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QACD,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QACrE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAA;IAEV,iBAAiB;IACjB,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAA;IACxD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;KACnB,CAAA;IACD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IAErE,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAC7G,CAAA;AACF,CAAC;AAcD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,GAAG,CAAC;QAClE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC"}
|
|
1
|
+
{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../../src/tools/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAG5D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,+CAA+C;AAE/C,MAAM,KAAK,GAAG,EAAE,CAAA;AAChB,MAAM,YAAY,GAAG,EAAE,CAAA;AAEvB,+FAA+F;AAC/F,kGAAkG;AAClG,mGAAmG;AACnG,mGAAmG;AACnG,+FAA+F;AAC/F,oEAAoE;AACpE,SAAS,YAAY,CAAC,IAAY,EAAE,YAAoB,KAAK;IAC5D,MAAM,MAAM,GAAa,EAAE,CAAA;IAC3B,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,QAAQ,GAAG,KAAK,CAAA;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;QAE7B,IAAI,EAAE,KAAK,YAAY,EAAE,CAAC;YACzB,QAAQ,GAAG,CAAC,QAAQ,CAAA;QACrB,CAAC;aAAM,IAAI,EAAE,KAAK,SAAS,IAAI,CAAC,QAAQ,EAAE,CAAC;YAC1C,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAA;YACjC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAA;IAE9B,OAAO,MAAM,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IACjC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IAE5B,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC7E,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IAClD,CAAC;IAED,OAAO,OAAO,CAAA;AACf,CAAC;AAED,YAAY;AAEZ,mCAAmC;AAEnC,SAAS,mBAAmB,CAAC,GAAW;IACvC,OAAO,CACN,GAAG;SACD,IAAI,EAAE;SACN,UAAU,CAAC,UAAU,EAAE,EAAE,CAAC;SAC1B,WAAW,EAAE;SACb,UAAU,CAAC,aAAa,EAAE,GAAG,CAAC;SAC9B,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,IAAI,SAAS,CACvC,CAAA;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAe;IACpC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAkB,CAAA;IAEtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,GAAG,CAAC,CAAC,CAAA;QAEzB,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,KAAK,GAAG,CAAC,EAAE,CAAA;IACnD,CAAC,CAAC,CAAA;AACH,CAAC;AAcD,SAAS,cAAc,CAAC,GAAW;IAClC,IAAI,CAAC,GAAG,GAAG,CAAC,IAAI,EAAE,CAAA;IAElB,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3D,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;IACzC,CAAC;IAED,oCAAoC;IACpC,IAAI,CAAC,CAAC,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,MAAM,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,WAAW,EAAE,CAAC;QACxG,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED,SAAS,eAAe,CAAC,OAA0B;IAClD,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACzB,IAAI,CAAC,KAAK,IAAI,EAAE,CAAC;YAChB,SAAS,EAAE,CAAA;YAEX,SAAQ;QACT,CAAC;QAED,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACvB,QAAQ,EAAE,CAAA;QACX,CAAC;aAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,SAAS,EAAE,CAAA;QACZ,CAAC;aAAM,CAAC;YACP,SAAS,EAAE,CAAA;QACZ,CAAC;IACF,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAA;IAC5B,MAAM,IAAI,GAAkB,SAAS,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;IAE5G,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,GAAG,KAAK,IAAI,GAAG,EAAE,CAAA;AAC9D,CAAC;AAiBD,KAAK,UAAU,SAAS,CAAC,IAAmB;IAC3C,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;IAExC,+DAA+D;IAC/D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,IAAI,CAAC,SAAS,4BAA4B,CAAC,CAAA;IAE3E,iGAAiG;IACjG,qGAAqG;IACrG,qGAAqG;IACrG,sGAAsG;IACtG,8BAA8B;IAC9B,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,MAAM,UAAU,GAAe,EAAE,CAAA;IACjC,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,2BAA2B;QAC3B,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnC,UAAU,GAAG,IAAI,CAAA;YAEjB,SAAQ;QACT,CAAC;QAED,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACzC,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;QAC1D,CAAC;aAAM,CAAC;YACP,MAAK;QACN,CAAC;IACF,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAC/C,CAAC;IAED,iCAAiC;IACjC,IAAI,UAAoB,CAAA;IAExB,IAAI,IAAI,CAAC,SAAS,IAAI,UAAU,EAAE,CAAC;QAClC,UAAU,GAAG,YAAY,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5D,CAAC;SAAM,CAAC;QACP,gDAAgD;QAChD,MAAM,OAAO,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAA;QAC1C,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACnE,CAAC;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,UAAU,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,CAAA;IAElE,6BAA6B;IAC7B,MAAM,OAAO,GAAiB,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE;QACtD,MAAM,OAAO,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YACtC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAExB,OAAO,cAAc,CAAC,GAAG,CAAC,CAAA;QAC3B,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,eAAe,CAAC,OAAO,CAAC,CAAA;QACrC,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;QAEhB,OAAO,IAAI,CAAA;IACZ,CAAC,CAAC,CAAA;IAEF,uBAAuB;IACvB,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;IACzE,kGAAkG;IAClG,sGAAsG;IACtG,MAAM,cAAc,GAAG,+BAA+B,IAAI,CAAC,SAAS,UAAU,OAAO,MAAM,CAAA;IAE3F,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7D,MAAM,SAAS,GAAG,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,QAAQ,eAAe,IAAI,CAAC,SAAS,WAAW,CAAA;IAE3H,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,0BAA0B,UAAU,CAAC,MAAM,iBAAiB,CAAC,CAAA;IAClF,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,cAAc,MAAM,CAAC,CAAA;IAE7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QAE3D,OAAM;IACP,CAAC;IAED,mCAAmC;IACnC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAA;IACpD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAExD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAC5C,EAAE,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAA,CAAC,yBAAyB;IAC9D,EAAE,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAA;IAEnC,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IAEvB,mFAAmF;IACnF,wCAAwC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG;QACjB,sBAAsB,IAAI,CAAC,SAAS,aAAa,OAAO,IAAI;QAC5D,WAAW;QACX,eAAe,IAAI,CAAC,SAAS,GAAG;QAChC,YAAY,WAAW,YAAY,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,kBAAkB,IAAI,CAAC,SAAS,SAAS;QACrH,SAAS;QACT,oBAAoB,IAAI,CAAC,SAAS,WAAW;KAC7C,CAAA;IAED,mFAAmF;IACnF,yDAAyD;IACzD,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,CAAA;IAE3C,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC5B,gBAAgB,IAAI,CAAC,SAAS,MAAM,QAAQ,aAAa,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC7F,CAAA;IAED,IAAI,QAAQ,GAAG,CAAC,CAAA;IAChB,IAAI,aAAa,GAAG,KAAK,CAAA;IAEzB,0FAA0F;IAC1F,MAAM,QAAQ,GAAG,GAAG,EAAE;QACrB,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QAEhB,IAAI,CAAC;YACJ,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,CAAA;YACvB,CAAC;YAED,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;YACnB,MAAM,KAAK,CAAA;QACZ,CAAC;IACF,CAAC,CAAA;IAED,MAAM,KAAK,GAAsB,EAAE,CAAA;IACnC,MAAM,UAAU,GAAG,MAAM,CAAA;IAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,CAAA;QAET,IAAI,OAAO,IAAI,IAAI,CAAC,SAAS;YAAE,SAAQ;QAEvC,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;YACtC,aAAa,GAAG,IAAI,CAAA;YAEpB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,YAAY,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;QAEvD,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,CAAA;YAE3B,IAAI,CAAC,KAAK,IAAI;gBAAE,OAAO,IAAI,CAAA;YAC3B,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;YAEtB,IAAI,GAAG,EAAE,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAE/E,IAAI,GAAG,EAAE,IAAI,KAAK,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC;gBAAE,OAAO,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;YAEhF,OAAO,CAAC,CAAA;QACT,CAAC,CAAC,CAAA;QAEF,kCAAkC;QAClC,OAAO,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClB,CAAC;QAED,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAA;QAE9B,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAElB,IAAI,KAAK,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YAChC,QAAQ,EAAE,CAAA;YACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;YACxB,KAAK,CAAC,MAAM,GAAG,CAAC,CAAA;YAEhB,IAAI,QAAQ,GAAG,OAAO,KAAK,CAAC,EAAE,CAAC;gBAC9B,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,GAAG,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,CAAA;YAC1E,CAAC;QACF,CAAC;IACF,CAAC;IAED,kBAAkB;IAClB,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAClB,QAAQ,EAAE,CAAA;QACV,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAA;IACzB,CAAC;IAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,cAAc,QAAQ,CAAC,cAAc,EAAE,eAAe,IAAI,CAAC,SAAS,KAAK,CAAC,CAAA;IAE/F,wEAAwE;IACxE,MAAM,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAA;IAE3D,IAAI,YAAY,EAAE,CAAC;QAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,YAAY,CAAC,IAAI,QAAQ,CAAC,CAAA;QAErE,EAAE,CAAC,IAAI,CACN,kCAAkC,IAAI,CAAC,SAAS,IAAI,YAAY,CAAC,IAAI,QAAQ,IAAI,CAAC,SAAS,MAAM,YAAY,CAAC,IAAI,KAAK,CACvH,CAAA;IACF,CAAC;IAED,EAAE,CAAC,KAAK,EAAE,CAAA;IAEV,iBAAiB;IACjB,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAA;IACxD,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAA;IAE3E,MAAM,QAAQ,GAAG;QAChB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,UAAU,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC;QACpC,UAAU,EAAE,IAAI,CAAC,SAAS;QAC1B,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;QACnF,SAAS,EAAE,QAAQ;QACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;KACnB,CAAA;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAA;IACvE,aAAa,CAAC,YAAY,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IAErE,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,SAAS,QAAQ,CAAC,cAAc,EAAE,WAAW,IAAI,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAC7G,CAAA;AACF,CAAC;AAgBD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACxD,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CAAC,mBAAmB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IACpD,CAAC;IAED,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAE/D,MAAM,SAAS,CAAC;QACf,SAAS,EAAE,OAAO,CAAC,KAAK;QACxB,SAAS,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,UAAU,CAAC,gBAAgB,EAAE,GAAG,CAAC;QACrE,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,KAAK,CAAC;QAC3E,UAAU,EAAE,OAAO,CAAC,MAAM,IAAI,GAAG;QACjC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG;QACnC,SAAS,EAAE,OAAO,CAAC,IAAI,IAAI,CAAC;QAC5B,SAAS,EAAE,CAAC,OAAO,CAAC,QAAQ;QAC5B,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,KAAK;KAC/B,CAAC,CAAA;AACH,CAAC;AAED,YAAY"}
|
|
@@ -29,21 +29,33 @@
|
|
|
29
29
|
* Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
|
|
30
30
|
* /tmp/part-po-box.parquet
|
|
31
31
|
*/
|
|
32
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Options for {@linkcode jsonlToParquet}.
|
|
34
|
+
*/
|
|
33
35
|
export interface JSONLToParquetOptions {
|
|
34
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* The labeled-row JSONL to convert.
|
|
38
|
+
*/
|
|
35
39
|
input: string;
|
|
36
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* The parquet shard to write.
|
|
42
|
+
*/
|
|
37
43
|
output: string;
|
|
38
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Parquet row-group size. Default 50000.
|
|
46
|
+
*/
|
|
39
47
|
rowGroupSize?: number;
|
|
40
48
|
}
|
|
41
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Summary returned by {@linkcode jsonlToParquet}.
|
|
51
|
+
*/
|
|
42
52
|
export interface JSONLToParquetSummary {
|
|
43
53
|
read: number;
|
|
44
54
|
written: number;
|
|
45
55
|
outPath: string;
|
|
46
56
|
}
|
|
47
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
|
|
59
|
+
*/
|
|
48
60
|
export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
|
|
49
61
|
//# sourceMappingURL=jsonl-to-parquet.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH
|
|
1
|
+
{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AAqCD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CAmEhC"}
|
|
@@ -96,13 +96,17 @@ function assertSpanTriple(row, lineNo) {
|
|
|
96
96
|
`tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
/**
|
|
99
|
+
/**
|
|
100
|
+
* Escape a path for single-quoted SQL string literals.
|
|
101
|
+
*/
|
|
100
102
|
function sqlString(value) {
|
|
101
|
-
return value.
|
|
103
|
+
return value.replaceAll("'", "''");
|
|
102
104
|
}
|
|
103
|
-
/**
|
|
105
|
+
/**
|
|
106
|
+
* Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
|
|
107
|
+
*/
|
|
104
108
|
export async function jsonlToParquet(options, report) {
|
|
105
|
-
const rowGroupSize = options.rowGroupSize ??
|
|
109
|
+
const rowGroupSize = options.rowGroupSize ?? 50_000;
|
|
106
110
|
if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
|
|
107
111
|
throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
|
|
108
112
|
}
|
|
@@ -130,7 +134,9 @@ export async function jsonlToParquet(options, report) {
|
|
|
130
134
|
stage.write(line + "\n");
|
|
131
135
|
rows++;
|
|
132
136
|
}
|
|
133
|
-
await new Promise((resolve, reject) =>
|
|
137
|
+
await new Promise((resolve, reject) => {
|
|
138
|
+
stage.end((err) => (err ? reject(err) : resolve()));
|
|
139
|
+
});
|
|
134
140
|
report?.(`Read ${rows} rows from ${options.input}`);
|
|
135
141
|
const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
|
|
136
142
|
const selectList = REQUIRED_COLUMNS.join(", ");
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;
|
|
1
|
+
{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,kGAAkG;IAClG,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,UAAU,EAAE,SAAS,CAAC,CAAA;IAC9E,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhE,IAAI,CAAC;QACJ,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,8FAA8F;QAC9F,iGAAiG;QACjG,6FAA6F;QAC7F,6CAA6C;QAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAA;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;YAE3B,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA4B,CAAA;YACvD,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC7B,6FAA6F;YAC7F,yFAAyF;YACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;YAExB,IAAI,EAAE,CAAA;QACP,CAAC;QAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QACnE,CAAC,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;QACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE9C,gFAAgF;QAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;QAC9C,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QACnC,8FAA8F;QAC9F,6FAA6F;QAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,SAAS,CAAC,SAAS,CAAC,KAAK;YACtE,aAAa,cAAc,mCAAmC;YAC9D,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACzG,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAChH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,KAAK,CAAC,OAAO,EAAE,CAAA;QACf,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACxC,CAAC;AACF,CAAC"}
|
|
@@ -33,34 +33,58 @@
|
|
|
33
33
|
* [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
|
|
34
34
|
* 50]
|
|
35
35
|
*/
|
|
36
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Options for {@linkcode lintShardVocab}.
|
|
38
|
+
*/
|
|
37
39
|
export interface LintShardVocabOptions {
|
|
38
|
-
/**
|
|
40
|
+
/**
|
|
41
|
+
* The shard parquet to lint.
|
|
42
|
+
*/
|
|
39
43
|
shard: string;
|
|
40
|
-
/**
|
|
44
|
+
/**
|
|
45
|
+
* Base corpus version. Default `v0.5.0`.
|
|
46
|
+
*/
|
|
41
47
|
baseVersion?: string;
|
|
42
|
-
/**
|
|
48
|
+
/**
|
|
49
|
+
* Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
|
|
50
|
+
*/
|
|
43
51
|
baseRoot?: string;
|
|
44
|
-
/**
|
|
52
|
+
/**
|
|
53
|
+
* Base-majority confidence floor for a contradiction. Default 0.7.
|
|
54
|
+
*/
|
|
45
55
|
threshold?: number;
|
|
46
|
-
/**
|
|
56
|
+
/**
|
|
57
|
+
* Minimum base support to judge a token. Default 50.
|
|
58
|
+
*/
|
|
47
59
|
minCount?: number;
|
|
48
|
-
/**
|
|
60
|
+
/**
|
|
61
|
+
* Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
|
|
62
|
+
*/
|
|
49
63
|
fraction?: number;
|
|
50
64
|
}
|
|
51
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* One contradiction row: token, shard tag, base tag, base fraction, base total.
|
|
67
|
+
*/
|
|
52
68
|
export type ShardVocabRow = [token: string, shardTag: string, baseTag: string, baseFrac: number, baseTotal: number];
|
|
53
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Findings summary returned by {@linkcode lintShardVocab}.
|
|
71
|
+
*/
|
|
54
72
|
export interface LintShardVocabSummary {
|
|
55
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Real contradictions — the command exits 1 when nonzero.
|
|
75
|
+
*/
|
|
56
76
|
errors: number;
|
|
57
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
|
|
79
|
+
*/
|
|
58
80
|
warnings: number;
|
|
59
81
|
findings: {
|
|
60
82
|
contradictions: ShardVocabRow[];
|
|
61
83
|
affixSplits: ShardVocabRow[];
|
|
62
84
|
};
|
|
63
85
|
}
|
|
64
|
-
/**
|
|
86
|
+
/**
|
|
87
|
+
* Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
|
|
88
|
+
*/
|
|
65
89
|
export declare function lintShardVocab(options: LintShardVocabOptions): Promise<LintShardVocabSummary>;
|
|
66
90
|
//# sourceMappingURL=lint-shard-vocab.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lint-shard-vocab.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;
|
|
1
|
+
{"version":3,"file":"lint-shard-vocab.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA6KH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAmJnG"}
|
|
@@ -36,7 +36,9 @@
|
|
|
36
36
|
import { readdirSync } from "node:fs";
|
|
37
37
|
import { join } from "node:path";
|
|
38
38
|
import { dataRootPath } from "@mailwoman/core/utils";
|
|
39
|
-
/**
|
|
39
|
+
/**
|
|
40
|
+
* Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
|
|
41
|
+
*/
|
|
40
42
|
function stripBIO(label) {
|
|
41
43
|
const head = label.slice(0, 2);
|
|
42
44
|
return head === "B-" || head === "I-" ? label.slice(2) : label;
|
|
@@ -62,7 +64,9 @@ function pyRound(x) {
|
|
|
62
64
|
return floor + 1;
|
|
63
65
|
return floor % 2 === 0 ? floor : floor + 1;
|
|
64
66
|
}
|
|
65
|
-
/**
|
|
67
|
+
/**
|
|
68
|
+
* Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
|
|
69
|
+
*/
|
|
66
70
|
function pct(frac) {
|
|
67
71
|
return `${pyRound(frac * 100)}%`;
|
|
68
72
|
}
|
|
@@ -74,7 +78,9 @@ function pct(frac) {
|
|
|
74
78
|
function pyFloat(n) {
|
|
75
79
|
return Number.isInteger(n) ? n.toFixed(1) : String(n);
|
|
76
80
|
}
|
|
77
|
-
/**
|
|
81
|
+
/**
|
|
82
|
+
* Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
|
|
83
|
+
*/
|
|
78
84
|
function pad(value, width) {
|
|
79
85
|
return value.padEnd(width);
|
|
80
86
|
}
|
|
@@ -94,10 +100,12 @@ function dominant(counter) {
|
|
|
94
100
|
}
|
|
95
101
|
}
|
|
96
102
|
if (total === 0)
|
|
97
|
-
return ["", 0, 0
|
|
103
|
+
return ["", 0, 0];
|
|
98
104
|
return [bestTag, total, bestCount / total];
|
|
99
105
|
}
|
|
100
|
-
/**
|
|
106
|
+
/**
|
|
107
|
+
* Bump a (key -> count) tally, creating the inner counter on first sight.
|
|
108
|
+
*/
|
|
101
109
|
function bump(table, key, sub) {
|
|
102
110
|
let counter = table.get(key);
|
|
103
111
|
if (!counter) {
|
|
@@ -128,13 +136,17 @@ async function readRows(con, path) {
|
|
|
128
136
|
}
|
|
129
137
|
return rows;
|
|
130
138
|
}
|
|
131
|
-
/**
|
|
139
|
+
/**
|
|
140
|
+
* Read just the first row's `source` value — used to group base parts for a proportional slice.
|
|
141
|
+
*/
|
|
132
142
|
async function readSource(con, path) {
|
|
133
143
|
const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
|
|
134
144
|
const rows = result.getRowObjects();
|
|
135
145
|
return rows.length ? String(rows[0].source) : "";
|
|
136
146
|
}
|
|
137
|
-
/**
|
|
147
|
+
/**
|
|
148
|
+
* Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`.
|
|
149
|
+
*/
|
|
138
150
|
function globParquet(dir) {
|
|
139
151
|
let names;
|
|
140
152
|
try {
|
|
@@ -146,15 +158,17 @@ function globParquet(dir) {
|
|
|
146
158
|
return names
|
|
147
159
|
.filter((f) => f.endsWith(".parquet"))
|
|
148
160
|
.map((f) => join(dir, f))
|
|
149
|
-
.
|
|
161
|
+
.toSorted();
|
|
150
162
|
}
|
|
151
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
|
|
165
|
+
*/
|
|
152
166
|
export async function lintShardVocab(options) {
|
|
153
167
|
const baseVersion = options.baseVersion ?? "v0.5.0";
|
|
154
168
|
const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
|
|
155
169
|
const threshold = options.threshold ?? 0.7;
|
|
156
170
|
const minCount = options.minCount ?? 50;
|
|
157
|
-
const fraction = options.fraction ?? 1
|
|
171
|
+
const fraction = options.fraction ?? 1;
|
|
158
172
|
// @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
|
|
159
173
|
const { DuckDBInstance } = await import("@duckdb/node-api");
|
|
160
174
|
const instance = await DuckDBInstance.create();
|
|
@@ -187,7 +201,7 @@ export async function lintShardVocab(options) {
|
|
|
187
201
|
if (!parts.length) {
|
|
188
202
|
throw new Error("no base parts found");
|
|
189
203
|
}
|
|
190
|
-
if (fraction < 1
|
|
204
|
+
if (fraction < 1) {
|
|
191
205
|
const bysrc = new Map();
|
|
192
206
|
for (const p of parts) {
|
|
193
207
|
const src = await readSource(con, p);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lint-shard-vocab.js","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;
|
|
1
|
+
{"version":3,"file":"lint-shard-vocab.js","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAWpD;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAA;IAEtB,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAE5B,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,GAAG,CAAC,CAAA;IAEhC,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AASD;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAY,CAAA;QACtD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAY,CAAA;QAEtD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC/B,IAAI,KAAe,CAAA;IAEnB,IAAI,CAAC;QACJ,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;SACxB,QAAQ,EAAE,CAAA;AACb,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,gFAAgF;IAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IAEpC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9E,IAAI,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IAElH,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}
|
|
@@ -34,20 +34,34 @@
|
|
|
34
34
|
* [--out-md /tmp/lint-report.md]\
|
|
35
35
|
* [--out-json /tmp/lint-report.json]
|
|
36
36
|
*/
|
|
37
|
-
/**
|
|
37
|
+
/**
|
|
38
|
+
* Options for {@linkcode lintCorpusShard}.
|
|
39
|
+
*/
|
|
38
40
|
export interface LintCorpusShardOptions {
|
|
39
|
-
/**
|
|
41
|
+
/**
|
|
42
|
+
* The new shard parquet to lint.
|
|
43
|
+
*/
|
|
40
44
|
shardPath: string;
|
|
41
|
-
/**
|
|
45
|
+
/**
|
|
46
|
+
* Pre-computed corpus stats JSON (see `corpus-stats.ts`).
|
|
47
|
+
*/
|
|
42
48
|
statsPath: string;
|
|
43
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module.
|
|
51
|
+
*/
|
|
44
52
|
rulesPath?: string;
|
|
45
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Write the markdown report here as well as stdout.
|
|
55
|
+
*/
|
|
46
56
|
outMd?: string;
|
|
47
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Write a JSON sidecar of the flags + summary here.
|
|
59
|
+
*/
|
|
48
60
|
outJson?: string;
|
|
49
61
|
}
|
|
50
|
-
/**
|
|
62
|
+
/**
|
|
63
|
+
* One lint flag emitted by a check.
|
|
64
|
+
*/
|
|
51
65
|
export interface LintShardFlag {
|
|
52
66
|
check: string;
|
|
53
67
|
severity: "error" | "warn";
|
|
@@ -60,14 +74,20 @@ export interface LintShardFlag {
|
|
|
60
74
|
detail: string;
|
|
61
75
|
ruleID?: string;
|
|
62
76
|
}
|
|
63
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Findings summary returned by {@linkcode lintCorpusShard}.
|
|
79
|
+
*/
|
|
64
80
|
export interface LintCorpusShardSummary {
|
|
65
81
|
errors: number;
|
|
66
82
|
warnings: number;
|
|
67
83
|
findings: LintShardFlag[];
|
|
68
|
-
/**
|
|
84
|
+
/**
|
|
85
|
+
* The rendered markdown report (also printed to stdout).
|
|
86
|
+
*/
|
|
69
87
|
report: string;
|
|
70
88
|
}
|
|
71
|
-
/**
|
|
89
|
+
/**
|
|
90
|
+
* Lint a shard against corpus stats + the anti-pattern rules; print the markdown report to stdout.
|
|
91
|
+
*/
|
|
72
92
|
export declare function lintCorpusShard(options: LintCorpusShardOptions, report?: (line: string) => void): LintCorpusShardSummary;
|
|
73
93
|
//# sourceMappingURL=lint-shard.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"lint-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;
|
|
1
|
+
{"version":3,"file":"lint-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AA4CH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAkID;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAqND;;GAEG;AACH,wBAAgB,eAAe,CAC9B,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,sBAAsB,CAkExB"}
|