@mailwoman/corpus 8.0.0 → 8.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +15 -5
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +11 -9
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +7 -0
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +9 -2
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +5 -1
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +14 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +15 -2
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +11 -0
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +24 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +11 -0
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +14 -1
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +10 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -2
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +18 -6
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +16 -6
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +11 -0
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +18 -3
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts +11 -0
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +12 -1
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +11 -0
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +11 -0
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +11 -0
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +11 -0
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +11 -0
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +15 -2
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +11 -0
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +13 -2
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts +14 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +19 -4
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +12 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +11 -0
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +17 -2
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +12 -1
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +11 -0
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +11 -0
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +15 -2
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +7 -0
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +18 -3
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +13 -3
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +15 -5
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +7 -0
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +9 -2
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +12 -4
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +9 -5
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +24 -8
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +3 -3
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -2
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +3 -1
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts +21 -7
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +18 -10
- package/out/src/golden.js.map +1 -1
- package/out/src/license.d.ts +3 -1
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +4 -2
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +4 -2
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +5 -4
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +14 -10
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js +6 -2
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +12 -4
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +12 -4
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +21 -7
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +15 -4
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +39 -13
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +7 -3
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +4 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +14 -5
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +4 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -1
- package/out/src/shard-recipes/boundary-stress.js +13 -7
- package/out/src/shard-recipes/boundary-stress.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +4 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +39 -11
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts.map +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.js +4 -0
- package/out/src/shard-recipes/cz-pcfirst-preposition.js.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +4 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-admin-split.js +14 -3
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +4 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-bare-street.js +4 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts +4 -0
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +25 -8
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.d.ts +4 -0
- package/out/src/shard-recipes/fr-lieudit.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-lieudit.js +21 -8
- package/out/src/shard-recipes/fr-lieudit.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +4 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +27 -8
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts +4 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +16 -6
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +4 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -1
- package/out/src/shard-recipes/house-venue.js +4 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts +9 -3
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +12 -4
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +4 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +27 -13
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts +4 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +9 -4
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/nl-postcode.d.ts +4 -0
- package/out/src/shard-recipes/nl-postcode.d.ts.map +1 -1
- package/out/src/shard-recipes/nl-postcode.js +8 -2
- package/out/src/shard-recipes/nl-postcode.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts +4 -0
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -2
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +4 -0
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +5 -1
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/no-street.d.ts +4 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street.js +4 -0
- package/out/src/shard-recipes/no-street.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +4 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +99 -43
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/po-box.d.ts +4 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box.js +4 -0
- package/out/src/shard-recipes/po-box.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +48 -16
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +10 -6
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +4 -0
- package/out/src/shard-recipes/si-bare-village.d.ts.map +1 -1
- package/out/src/shard-recipes/si-bare-village.js +4 -0
- package/out/src/shard-recipes/si-bare-village.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +4 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +29 -10
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +4 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -1
- package/out/src/shard-recipes/street-bare.js +4 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -1
- package/out/src/shard-recipes/street.d.ts +4 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -1
- package/out/src/shard-recipes/street.js +4 -0
- package/out/src/shard-recipes/street.js.map +1 -1
- package/out/src/shard-recipes/unit.d.ts +4 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +45 -14
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +18 -6
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +4 -2
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +7 -1
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +28 -14
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -1
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +35 -28
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -5
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +9 -3
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +15 -15
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +9 -3
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +25 -6
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +3 -1
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +5 -6
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -5
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +21 -7
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +3 -1
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +6 -1
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +39 -13
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +66 -36
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +15 -5
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -1
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +20 -8
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/ban.d.ts.map +1 -1
- package/out/src/tools/fetch/ban.js +17 -10
- package/out/src/tools/fetch/ban.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts +30 -10
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +26 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -1
- package/out/src/tools/fetch/hrsa.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +18 -18
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +3 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +3 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +21 -7
- package/out/src/tools/fetch/nad.d.ts.map +1 -1
- package/out/src/tools/fetch/nad.js +5 -5
- package/out/src/tools/fetch/nad.js.map +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +9 -15
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +3 -1
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -1
- package/out/src/tools/fetch/openaddresses.js +32 -23
- package/out/src/tools/fetch/openaddresses.js.map +1 -1
- package/out/src/tools/fetch/ppd.d.ts +3 -1
- package/out/src/tools/fetch/ppd.d.ts.map +1 -1
- package/out/src/tools/fetch/ppd.js +14 -3
- package/out/src/tools/fetch/ppd.js.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -1
- package/out/src/tools/fetch/state-hi-schools.js +13 -6
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -1
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -1
- package/out/src/tools/fetch/state-sources.js +8 -3
- package/out/src/tools/fetch/state-sources.js.map +1 -1
- package/out/src/tools/fetch/tiger-full.d.ts +9 -3
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -1
- package/out/src/tools/fetch/tiger-full.js +35 -20
- package/out/src/tools/fetch/tiger-full.js.map +1 -1
- package/out/src/tools/gb-title-case.d.ts.map +1 -1
- package/out/src/tools/gb-title-case.js.map +1 -1
- package/out/src/tools/golden-expand.d.ts +24 -8
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +20 -12
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts +18 -6
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +22 -7
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/ingest-csv.d.ts +3 -1
- package/out/src/tools/ingest-csv.d.ts.map +1 -1
- package/out/src/tools/ingest-csv.js +17 -19
- package/out/src/tools/ingest-csv.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts +18 -6
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +11 -5
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts +36 -12
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +25 -11
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +30 -10
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +33 -20
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +6 -4
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/shard-kryptonite.d.ts +6 -2
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +1 -1
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +9 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +2 -2
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/types.d.ts +51 -17
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +9 -3
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +6 -4
- package/out/src/wof-json.js.map +1 -1
- package/package.json +7 -6
- package/src/adapter.ts +24 -13
- package/src/adapters/ban/adapter.ts +12 -2
- package/src/adapters/ban/street-decompose.ts +7 -1
- package/src/adapters/fcc-bdc/adapter.ts +19 -3
- package/src/adapters/geonames/adapter.ts +28 -5
- package/src/adapters/geonames-postal/adapter.ts +17 -1
- package/src/adapters/gnaf/adapter.ts +15 -2
- package/src/adapters/gnaf/assemble.ts +40 -10
- package/src/adapters/index.ts +13 -0
- package/src/adapters/openaddresses/adapter.ts +22 -3
- package/src/adapters/overture/adapter.ts +18 -2
- package/src/adapters/state-hi-schools/adapter.ts +15 -0
- package/src/adapters/state-ia-contractors/adapter.ts +15 -0
- package/src/adapters/state-ny-notaries/adapter.ts +15 -0
- package/src/adapters/state-tx-notaries/adapter.ts +18 -2
- package/src/adapters/synth-po-box/adapter.ts +19 -2
- package/src/adapters/tiger/adapter.ts +23 -4
- package/src/adapters/tiger/street-decompose.ts +1 -0
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +18 -2
- package/src/adapters/usgov-imls-pls/adapter.ts +15 -0
- package/src/adapters/usgov-irs-bmf/adapter.ts +19 -2
- package/src/adapters/usgov-nad/adapter.ts +14 -1
- package/src/adapters/usgov-nppes/adapter.ts +13 -0
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +21 -3
- package/src/adapters/wof-admin-jp/adapter.ts +20 -3
- package/src/adapters/wof-admin-json/adapter.ts +22 -7
- package/src/adapters/wof-postalcode-json/adapter.ts +10 -2
- package/src/align.ts +21 -8
- package/src/build.ts +48 -11
- package/src/codex/us-fips-state.ts +6 -2
- package/src/golden.ts +28 -13
- package/src/license.ts +4 -2
- package/src/parquet-wrapper/reader.ts +6 -4
- package/src/parquet-wrapper/schema.ts +14 -10
- package/src/parquet-wrapper/writer.ts +12 -4
- package/src/parquet.ts +39 -8
- package/src/runner.ts +49 -15
- package/src/shard-recipes/anchor-absorption.ts +21 -5
- package/src/shard-recipes/boundary-stress.ts +20 -7
- package/src/shard-recipes/country-balanced.ts +78 -13
- package/src/shard-recipes/cz-pcfirst-preposition.ts +9 -0
- package/src/shard-recipes/fr-admin-split.ts +30 -5
- package/src/shard-recipes/fr-bare-street.ts +10 -0
- package/src/shard-recipes/fr-fragment.ts +33 -8
- package/src/shard-recipes/fr-lieudit.ts +35 -9
- package/src/shard-recipes/fr-order.ts +47 -9
- package/src/shard-recipes/german.ts +32 -6
- package/src/shard-recipes/house-venue.ts +7 -0
- package/src/shard-recipes/index.ts +12 -4
- package/src/shard-recipes/intersection.ts +63 -14
- package/src/shard-recipes/locale.ts +37 -4
- package/src/shard-recipes/nl-postcode.ts +12 -2
- package/src/shard-recipes/no-fragment.ts +12 -2
- package/src/shard-recipes/no-street-led.ts +11 -1
- package/src/shard-recipes/no-street.ts +7 -0
- package/src/shard-recipes/po-box-cedex.ts +145 -45
- package/src/shard-recipes/po-box.ts +7 -0
- package/src/shard-recipes/scaffold.ts +52 -20
- package/src/shard-recipes/si-bare-village.ts +9 -0
- package/src/shard-recipes/street-affix.ts +88 -13
- package/src/shard-recipes/street-bare.ts +10 -0
- package/src/shard-recipes/street.ts +7 -0
- package/src/shard-recipes/unit.ts +79 -16
- package/src/split.ts +28 -7
- package/src/synthesize-anchor-absorption.ts +41 -14
- package/src/synthesize-boundary-stress.ts +57 -30
- package/src/synthesize-german.ts +30 -7
- package/src/synthesize-house-venue.ts +22 -16
- package/src/synthesize-intersection.ts +34 -7
- package/src/synthesize-no-street.ts +18 -10
- package/src/synthesize-po-box.ts +42 -10
- package/src/synthesize-street.ts +10 -2
- package/src/synthesize.ts +81 -38
- package/src/tokenize.ts +15 -5
- package/src/tools/align-shard.ts +6 -1
- package/src/tools/audit.ts +57 -11
- package/src/tools/corpus-stats.ts +9 -0
- package/src/tools/fetch/ban.ts +32 -10
- package/src/tools/fetch/download.ts +49 -12
- package/src/tools/fetch/hrsa.ts +3 -0
- package/src/tools/fetch/imls-pls.ts +28 -18
- package/src/tools/fetch/index.ts +3 -1
- package/src/tools/fetch/nad.ts +39 -12
- package/src/tools/fetch/nppes.ts +17 -15
- package/src/tools/fetch/openaddresses.ts +52 -24
- package/src/tools/fetch/ppd.ts +22 -3
- package/src/tools/fetch/state-hi-schools.ts +17 -6
- package/src/tools/fetch/state-sources.ts +16 -3
- package/src/tools/fetch/tiger-full.ts +66 -23
- package/src/tools/gb-title-case.ts +1 -0
- package/src/tools/golden-expand.ts +64 -20
- package/src/tools/golden-promote.ts +53 -13
- package/src/tools/ingest-csv.ts +41 -26
- package/src/tools/jsonl-to-parquet.ts +30 -10
- package/src/tools/lint-shard-vocab.ts +75 -24
- package/src/tools/lint-shard.ts +75 -31
- package/src/tools/overlay-manifest.ts +7 -4
- package/src/tools/shard-kryptonite.ts +11 -3
- package/src/tools/shard-translit.ts +20 -5
- package/src/types.ts +51 -17
- package/src/wof-json.ts +18 -7
|
@@ -11,6 +11,8 @@ export interface PPDExtractStats {
|
|
|
11
11
|
* of rows (tests) or a streamed async source (the real 31M-row extraction).
|
|
12
12
|
*/
|
|
13
13
|
export declare function extractPPDTuples(input: AsyncIterable<string[]> | Iterable<string[]>, write: (line: string) => void): Promise<PPDExtractStats>;
|
|
14
|
-
/**
|
|
14
|
+
/**
|
|
15
|
+
* Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
|
|
16
|
+
*/
|
|
15
17
|
export declare function runPPDExtract(inputPath: string, outputPath: string): Promise<PPDExtractStats>;
|
|
16
18
|
//# sourceMappingURL=ppd.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AA2BA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,
|
|
1
|
+
{"version":3,"file":"ppd.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AA2BA,MAAM,WAAW,eAAe;IAC/B,IAAI,EAAE,MAAM,CAAA;IACZ,WAAW,EAAE,MAAM,CAAA;IACnB,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,iBAAiB,EAAE,MAAM,CAAA;CACzB;AAID;;;;GAIG;AACH,wBAAsB,gBAAgB,CACrC,KAAK,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EACnD,KAAK,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC3B,OAAO,CAAC,eAAe,CAAC,CAwD1B;AAED;;GAEG;AACH,wBAAsB,aAAa,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAiBnG"}
|
|
@@ -31,7 +31,14 @@ export async function extractPPDTuples(input, write) {
|
|
|
31
31
|
const stats = { kept: 0, skippedSAON: 0, skippedPAON: 0, skippedNoStreet: 0, skippedNoPostcode: 0 };
|
|
32
32
|
write("NUMBER,STREET,CITY,DISTRICT,REGION,POSTCODE");
|
|
33
33
|
for await (const cells of input) {
|
|
34
|
-
|
|
34
|
+
// HM Land Registry Price Paid Data column order (see the PPD FAQ for the full field list).
|
|
35
|
+
const postcode = cells[3];
|
|
36
|
+
const paon = cells[7];
|
|
37
|
+
const saon = cells[8];
|
|
38
|
+
const street = cells[9];
|
|
39
|
+
const locality = cells[10];
|
|
40
|
+
const town = cells[11];
|
|
41
|
+
const county = cells[13];
|
|
35
42
|
if (saon) {
|
|
36
43
|
stats.skippedSAON++;
|
|
37
44
|
continue;
|
|
@@ -62,7 +69,9 @@ export async function extractPPDTuples(input, write) {
|
|
|
62
69
|
}
|
|
63
70
|
return stats;
|
|
64
71
|
}
|
|
65
|
-
/**
|
|
72
|
+
/**
|
|
73
|
+
* Stream `inputPath` (PPD CSV) → `outputPath` (OA-shaped tuples CSV), returning the row-count stats.
|
|
74
|
+
*/
|
|
66
75
|
export async function runPPDExtract(inputPath, outputPath) {
|
|
67
76
|
// No `encoding` — CSVSpliterator delimits raw bytes and decodes utf-8 itself (see readTuples in
|
|
68
77
|
// shard-recipes/locale.ts). `header: false` yields every row as data — PPD ships no header row.
|
|
@@ -73,7 +82,9 @@ export async function runPPDExtract(inputPath, outputPath) {
|
|
|
73
82
|
});
|
|
74
83
|
const out = createWriteStream(outputPath, { encoding: "utf8" });
|
|
75
84
|
const stats = await extractPPDTuples(rows, (line) => out.write(line + "\n"));
|
|
76
|
-
await new Promise((res) =>
|
|
85
|
+
await new Promise((res) => {
|
|
86
|
+
out.end(res);
|
|
87
|
+
});
|
|
77
88
|
return stats;
|
|
78
89
|
}
|
|
79
90
|
runIfScript(import.meta, async () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,MAAM,
|
|
1
|
+
{"version":3,"file":"ppd.js","sourceRoot":"","sources":["../../../../src/tools/fetch/ppd.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAErC,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAA;AACvD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAE5C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAA;AAEjD,MAAM,oBAAoB,GAAG,sCAAsC,CAAA;AAUnE,MAAM,KAAK,GAAG,CAAC,KAAa,EAAU,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;AAE1F;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,KAAmD,EACnD,KAA6B;IAE7B,MAAM,KAAK,GAAoB,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,iBAAiB,EAAE,CAAC,EAAE,CAAA;IACpH,KAAK,CAAC,6CAA6C,CAAC,CAAA;IAEpD,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QACjC,2FAA2F;QAC3F,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACzB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAC1B,MAAM,IAAI,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QACtB,MAAM,MAAM,GAAG,KAAK,CAAC,EAAE,CAAC,CAAA;QAExB,IAAI,IAAI,EAAE,CAAC;YACV,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC/C,KAAK,CAAC,WAAW,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,MAAM,EAAE,CAAC;YACb,KAAK,CAAC,eAAe,EAAE,CAAA;YAEvB,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,KAAK,CAAC,iBAAiB,EAAE,CAAA;YAEzB,SAAQ;QACT,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAA;QAC3C,MAAM,IAAI,GAAG,QAAQ,IAAI,QAAQ,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;QAEvE,KAAK,CACJ;YACC,MAAM;YACN,KAAK,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YAC1B,KAAK,CAAC,IAAI,CAAC;YACX,KAAK,CAAC,WAAW,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;YAChC,QAAQ;SACR,CAAC,IAAI,CAAC,GAAG,CAAC,CACX,CAAA;QAED,KAAK,CAAC,IAAI,EAAE,CAAA;IACb,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,SAAiB,EAAE,UAAkB;IACxE,gGAAgG;IAChG,gGAAgG;IAChG,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAW,gBAAgB,CAAC,SAAS,CAAC,EAAE;QAC5E,IAAI,EAAE,OAAO;QACb,MAAM,EAAE,KAAK;QACb,mBAAmB,EAAE,IAAI;KACzB,CAAC,CAAA;IAEF,MAAM,GAAG,GAAG,iBAAiB,CAAC,UAAU,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAC/D,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAC,CAAA;IAE5E,MAAM,IAAI,OAAO,CAAO,CAAC,GAAG,EAAE,EAAE;QAC/B,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IACb,CAAC,CAAC,CAAA;IAEF,OAAO,KAAK,CAAA;AACb,CAAC;AAED,WAAW,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;IACnC,MAAM,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;QAC5B,OAAO,EAAE;YACR,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,iBAAiB,CAAC,EAAE;YACxF,MAAM,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,YAAY,CAAC,KAAK,EAAE,YAAY,EAAE,eAAe,CAAC,EAAE;SACvF;KACD,CAAC,CAAA;IAEF,MAAM,KAAK,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,KAAM,EAAE,MAAM,CAAC,MAAO,CAAC,CAAA;IAEhE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;AAC9C,CAAC,CAAC,CAAA"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"state-hi-schools.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AASH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAcnE,MAAM,MAAM,0BAA0B,GAAG,gBAAgB,CAAA;AA0FzD,wBAAsB,mBAAmB,CACxC,OAAO,EAAE,0BAA0B,EACnC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA2FvB"}
|
|
@@ -25,6 +25,11 @@ import { unlink } from "node:fs/promises";
|
|
|
25
25
|
import { join } from "node:path";
|
|
26
26
|
import { sha256File } from "@mailwoman/core/utils";
|
|
27
27
|
import { downloadToFile, readManifest, writeManifest } from "./download.js";
|
|
28
|
+
/**
|
|
29
|
+
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
30
|
+
* than data.
|
|
31
|
+
*/
|
|
32
|
+
const BYTES_PER_KIB = 1024;
|
|
28
33
|
const SOURCE_URL = "https://www.hawaiipublicschools.org/DOE%20Forms/SchoolList.xlsx";
|
|
29
34
|
const SLUG = "state-hi-schools";
|
|
30
35
|
const CSV_FILENAME = "HI_Public_Schools_List.csv";
|
|
@@ -72,14 +77,16 @@ with open(csv_path, "w", newline="", encoding="utf-8") as out:
|
|
|
72
77
|
|
|
73
78
|
print(f" converted {total_data_rows} data rows from {len(wb.sheetnames)} sheets", file=sys.stderr)
|
|
74
79
|
`;
|
|
75
|
-
/**
|
|
80
|
+
/**
|
|
81
|
+
* Mimic `numfmt --to=iec` for a friendly byte-size log line.
|
|
82
|
+
*/
|
|
76
83
|
function iec(bytes) {
|
|
77
|
-
if (bytes <
|
|
84
|
+
if (bytes < BYTES_PER_KIB)
|
|
78
85
|
return String(bytes);
|
|
79
86
|
const units = ["K", "M", "G", "T", "P"];
|
|
80
87
|
let value = bytes / 1024;
|
|
81
88
|
let i = 0;
|
|
82
|
-
while (value >=
|
|
89
|
+
while (value >= BYTES_PER_KIB && i < units.length - 1) {
|
|
83
90
|
value /= 1024;
|
|
84
91
|
i++;
|
|
85
92
|
}
|
|
@@ -136,13 +143,13 @@ export async function fetchStateHISchools(options, report) {
|
|
|
136
143
|
report,
|
|
137
144
|
});
|
|
138
145
|
}
|
|
139
|
-
catch (
|
|
140
|
-
report?.(` ✗ Download failed (${
|
|
146
|
+
catch (error) {
|
|
147
|
+
report?.(` ✗ Download failed (${error.message})`);
|
|
141
148
|
return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
|
|
142
149
|
}
|
|
143
150
|
const xlsxSize = statSync(xlsxDest).size;
|
|
144
151
|
report?.(` Downloaded XLSX: ${iec(xlsxSize)}`);
|
|
145
|
-
if (xlsxSize <
|
|
152
|
+
if (xlsxSize < BYTES_PER_KIB) {
|
|
146
153
|
report?.(` ✗ Response too small (${xlsxSize} bytes) — probable error page`);
|
|
147
154
|
return { fetched: 0, skipped: 0, failed: 1, failedCodes: [SLUG] };
|
|
148
155
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,YAAY,GAAG,4BAA4B,CAAA;AACjD,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAInD;;;GAGG;AACH,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClB,CAAA;AAWD
|
|
1
|
+
{"version":3,"file":"state-hi-schools.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-hi-schools.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AACrD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACzD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE3E;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,UAAU,GAAG,iEAAiE,CAAA;AACpF,MAAM,IAAI,GAAG,kBAAkB,CAAA;AAC/B,MAAM,YAAY,GAAG,4BAA4B,CAAA;AACjD,MAAM,aAAa,GAAG,6BAA6B,CAAA;AAInD;;;GAGG;AACH,MAAM,UAAU,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsClB,CAAA;AAWD;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa;IACzB,IAAI,KAAK,GAAG,aAAa;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAA;IAC/C,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAA;IACvC,IAAI,KAAK,GAAG,KAAK,GAAG,IAAI,CAAA;IACxB,IAAI,CAAC,GAAG,CAAC,CAAA;IAET,OAAO,KAAK,IAAI,aAAa,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvD,KAAK,IAAI,IAAI,CAAA;QAEb,CAAC,EAAE,CAAA;IACJ,CAAC;IAED,MAAM,OAAO,GAAG,KAAK,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAA;IAE5E,OAAO,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAA;AACrC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,QAAgB,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,EAAE;QACrE,KAAK,EAAE,CAAC,QAAQ,EAAE,SAAS,EAAE,SAAS,CAAC;KACvC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAClH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IAC1B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACxC,OAAmC,EACnC,MAA+B;IAE/B,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;IAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAA;IAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAA;IAC3C,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,CAAA;IAEnD,MAAM,EAAE,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEvB,qEAAqE;IACrE,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAoB,YAAY,CAAC,CAAA;QAEpE,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,KAAK,YAAY,EAAE,CAAC;YAC5D,MAAM,SAAS,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;YAE3C,IAAI,SAAS,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,EAAE,CAAC,oEAAoE,CAAC,CAAA;gBAE9E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;YAC9D,CAAC;QACF,CAAC;IACF,CAAC;IAED,0CAA0C;IAC1C,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,EAAE,CAAC,IAAI,EAAE,iBAAiB,CAAC,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEtF,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,MAAM,EAAE,CACP,oFAAoF;YACnF,gEAAgE;YAChE,kEAAkE,CACnE,CAAA;QAED,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,iBAAiB;IACjB,MAAM,EAAE,CAAC,iBAAiB,UAAU,MAAM,CAAC,CAAA;IAE3C,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC;YACpB,GAAG,EAAE,UAAU;YACf,IAAI,EAAE,QAAQ;YACd,SAAS,EAAE,OAAO;YAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;YAC1C,MAAM;SACN,CAAC,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,EAAE,CAAC,wBAAyB,KAAe,CAAC,OAAO,GAAG,CAAC,CAAA;QAE7D,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,MAAM,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAA;IACxC,MAAM,EAAE,CAAC,sBAAsB,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAA;IAE/C,IAAI,QAAQ,GAAG,aAAa,EAAE,CAAC;QAC9B,MAAM,EAAE,CAAC,2BAA2B,QAAQ,+BAA+B,CAAC,CAAA;QAE5E,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAA;IAClE,CAAC;IAED,wEAAwE;IACxE,MAAM,EAAE,CAAC,oDAAoD,CAAC,CAAA;IAC9D,MAAM,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAA;IAEzC,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,IAAI,CAAA;IACtC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,CAAA;IAExC,oEAAoE;IACpE,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAA;IACtB,MAAM,EAAE,CAAC,2BAA2B,CAAC,CAAA;IAErC,kBAAkB;IAClB,MAAM,QAAQ,GAAa;QAC1B,UAAU,EAAE,UAAU;QACtB,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACvC,QAAQ,EAAE,YAAY;QACtB,MAAM,EAAE,MAAM;QACd,KAAK,EAAE,OAAO;QACd,KAAK,EAAE,4EAA4E;KACnF,CAAA;IAED,MAAM,aAAa,CAAC,YAAY,EAAE,QAAQ,CAAC,CAAA;IAE3C,MAAM,EAAE,CAAC,OAAO,GAAG,CAAC,OAAO,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IACjD,MAAM,EAAE,CAAC,yBAAyB,YAAY,EAAE,CAAC,CAAA;IAEjD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,EAAE,EAAE,CAAA;AAC9D,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"state-sources.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAOH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AASnE,MAAM,MAAM,wBAAwB,GAAG,gBAAgB,CAAA;AAsDvD,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,wBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgEvB"}
|
|
@@ -20,6 +20,11 @@ import { mkdirSync } from "node:fs";
|
|
|
20
20
|
import { join } from "node:path";
|
|
21
21
|
import { sha256File } from "@mailwoman/core/utils";
|
|
22
22
|
import { downloadToFile, writeManifest } from "./download.js";
|
|
23
|
+
/**
|
|
24
|
+
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
25
|
+
* than data.
|
|
26
|
+
*/
|
|
27
|
+
const BYTES_PER_KIB = 1024;
|
|
23
28
|
const SOURCES = [
|
|
24
29
|
{
|
|
25
30
|
slug: "state-ny-notaries",
|
|
@@ -78,13 +83,13 @@ export async function fetchStateSources(options, report) {
|
|
|
78
83
|
report,
|
|
79
84
|
}));
|
|
80
85
|
}
|
|
81
|
-
catch (
|
|
82
|
-
report?.(` ✗ download failed for ${url}: ${
|
|
86
|
+
catch (error) {
|
|
87
|
+
report?.(` ✗ download failed for ${url}: ${error.message}`);
|
|
83
88
|
failed++;
|
|
84
89
|
failedCodes.push(slug);
|
|
85
90
|
continue;
|
|
86
91
|
}
|
|
87
|
-
if (bytes <
|
|
92
|
+
if (bytes < BYTES_PER_KIB) {
|
|
88
93
|
report?.(` ✗ response too small (${bytes} bytes) — probable 404 / error page`);
|
|
89
94
|
failed++;
|
|
90
95
|
failedCodes.push(slug);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"state-sources.js","sourceRoot":"","sources":["../../../../src/tools/fetch/state-sources.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAA;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE7D;;;GAGG;AACH,MAAM,aAAa,GAAG,IAAI,CAAA;AAU1B,MAAM,OAAO,GAAsB;IAClC;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,kCAAkC;QAC5C,GAAG,EAAE,yEAAyE;KAC9E;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,8BAA8B;QACxC,GAAG,EAAE,4EAA4E;KACjF;IACD;QACC,IAAI,EAAE,mBAAmB;QACzB,QAAQ,EAAE,wBAAwB;QAClC,GAAG,EAAE,0EAA0E;KAC/E;IACD;QACC,IAAI,EAAE,sBAAsB;QAC5B,QAAQ,EAAE,qDAAqD;QAC/D,GAAG,EAAE,wEAAwE;KAC7E;IACD;QACC,IAAI,EAAE,2BAA2B;QACjC,QAAQ,EAAE,6CAA6C;QACvD,GAAG,EAAE,sEAAsE;KAC3E;IACD;QACC,IAAI,EAAE,oBAAoB;QAC1B,QAAQ,EAAE,yCAAyC;QACnD,GAAG,EAAE,0EAA0E;KAC/E;CACD,CAAA;AAUD,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACtC,OAAiC,EACjC,MAA+B;IAE/B,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE/C,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,KAAK,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,OAAO,EAAE,CAAC;QAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,CAAA;QAC3C,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAA;QAEpC,MAAM,EAAE,CAAC,OAAO,IAAI,MAAM,QAAQ,EAAE,CAAC,CAAA;QAErC,IAAI,KAAa,CAAA;QAEjB,IAAI,CAAC;YACJ,CAAC;YAAA,CAAC,EAAE,KAAK,EAAE,GAAG,MAAM,cAAc,CAAC;gBAClC,GAAG;gBACH,IAAI;gBACJ,SAAS,EAAE,OAAO;gBAClB,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;gBAC1C,MAAM;aACN,CAAC,CAAC,CAAA;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,EAAE,CAAC,2BAA2B,GAAG,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YAEvE,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;YAC3B,MAAM,EAAE,CAAC,2BAA2B,KAAK,qCAAqC,CAAC,CAAA;YAE/E,MAAM,EAAE,CAAA;YACR,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEtB,SAAQ;QACT,CAAC;QAED,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;QAElC,MAAM,QAAQ,GAAmB;YAChC,UAAU,EAAE,GAAG;YACf,aAAa,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACvC,QAAQ;YACR,MAAM,EAAE,GAAG;YACX,KAAK;SACL,CAAA;QAED,MAAM,aAAa,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,CAAC,EAAE,QAAQ,CAAC,CAAA;QAE7D,MAAM,EAAE,CAAC,OAAO,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,GAAG,EAAE,CAAC,CAAA;QAErE,OAAO,EAAE,CAAA;IACV,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,YAAY,OAAO,EAAE,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9B,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,CAAA;AACpD,CAAC"}
|
|
@@ -25,11 +25,17 @@ export interface FetchTigerFullOptions extends BaseFetchOptions {
|
|
|
25
25
|
* v0.1.1.
|
|
26
26
|
*/
|
|
27
27
|
skipStateFips?: string;
|
|
28
|
-
/**
|
|
28
|
+
/**
|
|
29
|
+
* Seconds to sleep between downloads. Default `0.2`.
|
|
30
|
+
*/
|
|
29
31
|
rateSleep?: number;
|
|
30
|
-
/**
|
|
32
|
+
/**
|
|
33
|
+
* Max concurrent download workers per state. Default `4`.
|
|
34
|
+
*/
|
|
31
35
|
maxParallel?: number;
|
|
32
|
-
/**
|
|
36
|
+
/**
|
|
37
|
+
* Print planned downloads without fetching. Default `false`.
|
|
38
|
+
*/
|
|
33
39
|
dryRun?: boolean;
|
|
34
40
|
}
|
|
35
41
|
export declare function fetchTigerFull(options: FetchTigerFullOptions, report?: (line: string) => void): Promise<FetchSummary>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;
|
|
1
|
+
{"version":3,"file":"tiger-full.d.ts","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAaH,OAAO,KAAK,EAAE,gBAAgB,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAqBnE,MAAM,WAAW,qBAAsB,SAAQ,gBAAgB;IAC9D;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAwHD,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAsKvB"}
|
|
@@ -18,6 +18,8 @@
|
|
|
18
18
|
* Invoke via `mailwoman corpus fetch tiger-full --out-root <path>`. Native `fetch` streams each
|
|
19
19
|
* county ZIP to disk (no curl subprocess).
|
|
20
20
|
*/
|
|
21
|
+
/* oxlint-disable sister-software/prefer-region-over-marks -- these markers label steps inside one
|
|
22
|
+
procedure, not sections of declarations. A region there folds nothing a reader wants folded. */
|
|
21
23
|
import { createWriteStream, existsSync, mkdirSync, rmSync, statSync } from "node:fs";
|
|
22
24
|
import { basename, join } from "node:path";
|
|
23
25
|
import { Readable } from "node:stream";
|
|
@@ -25,12 +27,25 @@ import { pipeline } from "node:stream/promises";
|
|
|
25
27
|
import { setTimeout as sleep } from "node:timers/promises";
|
|
26
28
|
import { sha256File } from "@mailwoman/core/utils";
|
|
27
29
|
import { isTransientStatus, readManifest, writeManifest } from "./download.js";
|
|
30
|
+
/**
|
|
31
|
+
* Bytes per KiB — the divisor for human-readable sizes, and the floor below which a "download" is an error page rather
|
|
32
|
+
* than data.
|
|
33
|
+
*/
|
|
34
|
+
/**
|
|
35
|
+
* Lowest 2xx status; anything below is informational.
|
|
36
|
+
*/
|
|
37
|
+
const HTTP_OK = 200;
|
|
38
|
+
/**
|
|
39
|
+
* Lowest 3xx status; at or above it the response is a redirect or an error, not a body.
|
|
40
|
+
*/
|
|
41
|
+
const HTTP_REDIRECT = 300;
|
|
42
|
+
const BYTES_PER_KIB = 1024;
|
|
28
43
|
const TIGER_BASE_URL = "https://www2.census.gov/geo/tiger/TIGER2024/ADDRFEAT";
|
|
29
44
|
function humanBytes(bytes) {
|
|
30
45
|
const units = ["B", "KiB", "MiB", "GiB", "TiB"];
|
|
31
46
|
let value = bytes;
|
|
32
47
|
let unit = 0;
|
|
33
|
-
while (value >=
|
|
48
|
+
while (value >= BYTES_PER_KIB && unit < units.length - 1) {
|
|
34
49
|
value /= 1024;
|
|
35
50
|
unit++;
|
|
36
51
|
}
|
|
@@ -67,7 +82,9 @@ async function streamDownload(url, dest, opts) {
|
|
|
67
82
|
}
|
|
68
83
|
return 0;
|
|
69
84
|
}
|
|
70
|
-
/**
|
|
85
|
+
/**
|
|
86
|
+
* Read a per-state MANIFEST.json into a filename → entry map.
|
|
87
|
+
*/
|
|
71
88
|
async function readCountyManifest(manifestPath) {
|
|
72
89
|
const map = new Map();
|
|
73
90
|
const parsed = await readManifest(manifestPath);
|
|
@@ -78,7 +95,9 @@ async function readCountyManifest(manifestPath) {
|
|
|
78
95
|
}
|
|
79
96
|
return map;
|
|
80
97
|
}
|
|
81
|
-
/**
|
|
98
|
+
/**
|
|
99
|
+
* Check whether a file already matches a recorded sha256 and byte count.
|
|
100
|
+
*/
|
|
82
101
|
async function fileMatchesSha(path, expectedSha, expectedBytes) {
|
|
83
102
|
if (!existsSync(path))
|
|
84
103
|
return false;
|
|
@@ -86,15 +105,17 @@ async function fileMatchesSha(path, expectedSha, expectedBytes) {
|
|
|
86
105
|
return false;
|
|
87
106
|
return (await sha256File(path)) === expectedSha;
|
|
88
107
|
}
|
|
89
|
-
/**
|
|
108
|
+
/**
|
|
109
|
+
* Download one county ZIP (size sanity check + sha256).
|
|
110
|
+
*/
|
|
90
111
|
async function downloadCounty(url, dest) {
|
|
91
112
|
const filename = basename(dest);
|
|
92
|
-
const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs:
|
|
93
|
-
if (status <
|
|
113
|
+
const status = await streamDownload(url, dest, { timeoutMs: 600_000, retries: 3, retryDelayMs: 5000 });
|
|
114
|
+
if (status < HTTP_OK || status >= HTTP_REDIRECT) {
|
|
94
115
|
return { ok: false, filename, reason: `HTTP ${status}` };
|
|
95
116
|
}
|
|
96
117
|
const bytes = statSync(dest).size;
|
|
97
|
-
if (bytes <
|
|
118
|
+
if (bytes < BYTES_PER_KIB) {
|
|
98
119
|
rmSync(dest, { force: true });
|
|
99
120
|
return { ok: false, filename, reason: `too small (${bytes} bytes)` };
|
|
100
121
|
}
|
|
@@ -108,9 +129,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
108
129
|
const dryRun = options.dryRun ?? false;
|
|
109
130
|
const addrfeatDir = join(options.outRoot, "tiger", "addrfeat");
|
|
110
131
|
mkdirSync(addrfeatDir, { recursive: true });
|
|
111
|
-
//
|
|
112
|
-
// Step 1: Discover the full county file list from the TIGER directory listing.
|
|
113
|
-
// -------------------------------------------------------------------------
|
|
132
|
+
// MARK: Step 1 — discover the county file list
|
|
114
133
|
report?.(`=== Fetching TIGER 2024 ADDRFEAT directory listing...`);
|
|
115
134
|
const listingRes = await fetch(`${TIGER_BASE_URL}/`, {
|
|
116
135
|
headers: { "Accept-Encoding": "gzip, br" },
|
|
@@ -119,7 +138,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
119
138
|
if (!listingRes.ok)
|
|
120
139
|
throw new Error(`Failed to fetch TIGER directory listing: HTTP ${listingRes.status}`);
|
|
121
140
|
const html = await listingRes.text();
|
|
122
|
-
const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g)
|
|
141
|
+
const allZips = [...new Set(html.match(/tl_2024_[0-9]{5}_addrfeat\.zip/g))].toSorted();
|
|
123
142
|
const totalCounties = allZips.length;
|
|
124
143
|
report?.(` Found ${totalCounties} county ZIPs in the TIGER 2024 ADDRFEAT index.`);
|
|
125
144
|
// Build a map: state_fips -> list of filenames.
|
|
@@ -132,9 +151,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
132
151
|
stateFiles.set(stateFips, list);
|
|
133
152
|
}
|
|
134
153
|
report?.(` Spans ${stateFiles.size} state/territory FIPS codes.`);
|
|
135
|
-
//
|
|
136
|
-
// Step 2: For each state, download missing/unverified county ZIPs.
|
|
137
|
-
// -------------------------------------------------------------------------
|
|
154
|
+
// MARK: Step 2 — download the county ZIPs
|
|
138
155
|
let totalFetched = 0;
|
|
139
156
|
let totalSkipped = 0;
|
|
140
157
|
let totalSkippedState = 0;
|
|
@@ -142,7 +159,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
142
159
|
let totalBytesFetched = 0;
|
|
143
160
|
const failedCodes = [];
|
|
144
161
|
// Process states in sorted FIPS order for predictable output.
|
|
145
|
-
const sortedStates = [...stateFiles.keys()].
|
|
162
|
+
const sortedStates = [...stateFiles.keys()].toSorted();
|
|
146
163
|
for (const stateFips of sortedStates) {
|
|
147
164
|
const countyFiles = stateFiles.get(stateFips) ?? [];
|
|
148
165
|
// --- Skip entire state if requested ------------------------------------------
|
|
@@ -178,7 +195,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
178
195
|
}
|
|
179
196
|
if (dryRun)
|
|
180
197
|
continue;
|
|
181
|
-
if (pending.length
|
|
198
|
+
if (!pending.length)
|
|
182
199
|
continue;
|
|
183
200
|
// --- Download pending files with bounded parallelism + rate-limit spacing ---
|
|
184
201
|
const results = new Array(pending.length);
|
|
@@ -210,7 +227,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
210
227
|
}
|
|
211
228
|
}
|
|
212
229
|
// Rewrite per-state MANIFEST.json with all known-good counties (sorted for determinism).
|
|
213
|
-
const counties = [...manifest.values()].
|
|
230
|
+
const counties = [...manifest.values()].toSorted((a, b) => a.filename.localeCompare(b.filename));
|
|
214
231
|
const manifestDoc = {
|
|
215
232
|
state_fips: stateFips,
|
|
216
233
|
updated_at: new Date().toISOString().replace(/\.\d{3}Z$/, "Z"),
|
|
@@ -219,9 +236,7 @@ export async function fetchTigerFull(options, report) {
|
|
|
219
236
|
};
|
|
220
237
|
await writeManifest(manifestPath, manifestDoc);
|
|
221
238
|
}
|
|
222
|
-
//
|
|
223
|
-
// Summary
|
|
224
|
-
// -------------------------------------------------------------------------
|
|
239
|
+
// MARK: Summary
|
|
225
240
|
report?.(`=== Summary ===`);
|
|
226
241
|
report?.(` Total counties in index : ${totalCounties}`);
|
|
227
242
|
report?.(` State(s) fully skipped : ${totalSkippedState} (--skip-state-fips "${skipStateFips.join(" ")}")`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACpF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E,MAAM,cAAc,GAAG,sDAAsD,CAAA;
|
|
1
|
+
{"version":3,"file":"tiger-full.js","sourceRoot":"","sources":["../../../../src/tools/fetch/tiger-full.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;kGACkG;AAElG,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACpF,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAC1C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,UAAU,IAAI,KAAK,EAAE,MAAM,sBAAsB,CAAA;AAE1D,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAGlD,OAAO,EAAE,iBAAiB,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAE9E;;;GAGG;AACH;;GAEG;AACH,MAAM,OAAO,GAAG,GAAG,CAAA;AAEnB;;GAEG;AACH,MAAM,aAAa,GAAG,GAAG,CAAA;AAEzB,MAAM,aAAa,GAAG,IAAI,CAAA;AAE1B,MAAM,cAAc,GAAG,sDAAsD,CAAA;AA4B7E,SAAS,UAAU,CAAC,KAAa;IAChC,MAAM,KAAK,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAC/C,IAAI,KAAK,GAAG,KAAK,CAAA;IACjB,IAAI,IAAI,GAAG,CAAC,CAAA;IAEZ,OAAO,KAAK,IAAI,aAAa,IAAI,IAAI,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1D,KAAK,IAAI,IAAI,CAAA;QAEb,IAAI,EAAE,CAAA;IACP,CAAC;IAED,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAA;AAC5D,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,cAAc,CAC5B,GAAW,EACX,IAAY,EACZ,IAAkE;IAElE,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;QAC1D,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAA;YAEjG,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC;gBACxB,MAAM,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAA;gBAEnE,OAAO,GAAG,CAAC,MAAM,CAAA;YAClB,CAAC;YAED,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC7D,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,GAAG,CAAC,MAAM,CAAA;QAClB,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC;gBAC5B,MAAM,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAA;gBAE9B,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,CAAA;QACT,CAAC;IACF,CAAC;IAED,OAAO,CAAC,CAAA;AACT,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAAC,YAAoB;IACrD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAuB,CAAA;IAC1C,MAAM,MAAM,GAAG,MAAM,YAAY,CAA+B,YAAY,CAAC,CAAA;IAE7E,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,QAAQ,IAAI,EAAE,EAAE,CAAC;QACxC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YAChB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;QAChF,CAAC;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,IAAY,EAAE,WAAmB,EAAE,aAAqB;IACrF,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,KAAK,CAAA;IAEnC,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,aAAa;QAAE,OAAO,KAAK,CAAA;IAEvD,OAAO,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,KAAK,WAAW,CAAA;AAChD,CAAC;AAMD;;GAEG;AACH,KAAK,UAAU,cAAc,CAAC,GAAW,EAAE,IAAY;IACtD,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;IAC/B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,SAAS,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,YAAY,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtG,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,IAAI,aAAa,EAAE,CAAC;QACjD,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,MAAM,EAAE,EAAE,CAAA;IACzD,CAAC;IAED,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,IAAI,CAAA;IAEjC,IAAI,KAAK,GAAG,aAAa,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;QAE7B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,cAAc,KAAK,SAAS,EAAE,CAAA;IACrE,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;IAErC,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,CAAA;AAC7C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,aAAa,GAAG,CAAC,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC,GAAG,IAAI,CAAC,CAAA;IACjE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IAC5C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,EAAE,UAAU,CAAC,CAAA;IAC9D,SAAS,CAAC,WAAW,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,+CAA+C;IAE/C,MAAM,EAAE,CAAC,uDAAuD,CAAC,CAAA;IAEjE,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,GAAG,cAAc,GAAG,EAAE;QACpD,OAAO,EAAE,EAAE,iBAAiB,EAAE,UAAU,EAAE;QAC1C,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;KACnC,CAAC,CAAA;IAEF,IAAI,CAAC,UAAU,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,UAAU,CAAC,MAAM,EAAE,CAAC,CAAA;IACzG,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,IAAI,EAAE,CAAA;IACpC,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtF,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAA;IACpC,MAAM,EAAE,CAAC,WAAW,aAAa,gDAAgD,CAAC,CAAA;IAElF,gDAAgD;IAChD,uFAAuF;IACvF,MAAM,UAAU,GAAG,IAAI,GAAG,EAAoB,CAAA;IAE9C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;QACpC,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAC5C,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QAChB,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,EAAE,CAAC,WAAW,UAAU,CAAC,IAAI,8BAA8B,CAAC,CAAA;IAElE,0CAA0C;IAE1C,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,IAAI,iBAAiB,GAAG,CAAC,CAAA;IACzB,MAAM,WAAW,GAAa,EAAE,CAAA;IAEhC,8DAA8D;IAC9D,MAAM,YAAY,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEtD,KAAK,MAAM,SAAS,IAAI,YAAY,EAAE,CAAC;QACtC,MAAM,WAAW,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAA;QAEnD,gFAAgF;QAChF,IAAI,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YACvC,MAAM,EAAE,CAAC,aAAa,SAAS,qCAAqC,WAAW,CAAC,MAAM,YAAY,CAAC,CAAA;YACnG,iBAAiB,IAAI,WAAW,CAAC,MAAM,CAAA;YAEvC,SAAQ;QACT,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,WAAW,EAAE,SAAS,SAAS,EAAE,CAAC,CAAA;QACxD,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACxC,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAA;QAEpD,wDAAwD;QACxD,MAAM,QAAQ,GAAG,MAAM,kBAAkB,CAAC,YAAY,CAAC,CAAA;QAEvD,MAAM,EAAE,CAAC,aAAa,SAAS,MAAM,WAAW,CAAC,MAAM,WAAW,CAAC,CAAA;QAEnE,iDAAiD;QACjD,MAAM,OAAO,GAAyC,EAAE,CAAA;QAExD,KAAK,MAAM,KAAK,IAAI,WAAW,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAClC,MAAM,GAAG,GAAG,GAAG,cAAc,IAAI,KAAK,EAAE,CAAA;YACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,CAAA;YAEjC,yCAAyC;YACzC,IAAI,KAAK,IAAI,CAAC,MAAM,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;gBACtE,MAAM,EAAE,CAAC,qBAAqB,KAAK,EAAE,CAAC,CAAA;gBAEtC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,CAAC,kBAAkB,GAAG,EAAE,CAAC,CAAA;gBAEjC,YAAY,EAAE,CAAA;gBAEd,SAAQ;YACT,CAAC;YAED,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5B,CAAC;QAED,IAAI,MAAM;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,MAAM;YAAE,SAAQ;QAE7B,+EAA+E;QAC/E,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACzD,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;YACxF,OAAO,IAAI,EAAE,CAAC;gBACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;gBAElB,IAAI,CAAC,IAAI,OAAO,CAAC,MAAM;oBAAE,OAAM;gBAC/B,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;gBACxB,gDAAgD;gBAChD,MAAM,KAAK,CAAC,WAAW,CAAC,CAAA;gBACxB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,CAAA;YACvD,CAAC;QACF,CAAC,CAAC,CAAA;QAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QAE1B,mCAAmC;QACnC,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC9B,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACf,MAAM,EAAE,CAAC,QAAQ,MAAM,CAAC,QAAQ,KAAK,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAA;gBACzG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,CAAC,CAAA;gBAExG,YAAY,EAAE,CAAA;gBACd,iBAAiB,IAAI,MAAM,CAAC,KAAK,CAAA;YAClC,CAAC;iBAAM,CAAC;gBACP,MAAM,EAAE,CAAC,UAAU,MAAM,CAAC,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;gBAEzD,WAAW,EAAE,CAAA;gBACb,WAAW,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;YAClC,CAAC;QACF,CAAC;QAED,yFAAyF;QACzF,MAAM,QAAQ,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;QAEhG,MAAM,WAAW,GAAG;YACnB,UAAU,EAAE,SAAS;YACrB,UAAU,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;YAC9D,cAAc,EAAE,cAAc;YAC9B,QAAQ;SACR,CAAA;QAED,MAAM,aAAa,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAC/C,CAAC;IAED,gBAAgB;IAEhB,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,iCAAiC,aAAa,EAAE,CAAC,CAAA;IAC1D,MAAM,EAAE,CAAC,iCAAiC,iBAAiB,wBAAwB,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC/G,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,YAAY,EAAE,CAAC,CAAA;IACzD,MAAM,EAAE,CAAC,iCAAiC,WAAW,EAAE,CAAC,CAAA;IAExD,IAAI,iBAAiB,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,CAAC,iCAAiC,UAAU,CAAC,iBAAiB,CAAC,KAAK,iBAAiB,GAAG,CAAC,CAAA;IAClG,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,EAAE,CAAC,YAAY,WAAW,uCAAuC,CAAC,CAAA;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,YAAY,GAAG,iBAAiB,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,CAAA;AAC9G,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gb-title-case.d.ts","sourceRoot":"","sources":["../../../src/tools/gb-title-case.ts"],"names":[],"mappings":"AAkDA,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,
|
|
1
|
+
{"version":3,"file":"gb-title-case.d.ts","sourceRoot":"","sources":["../../../src/tools/gb-title-case.ts"],"names":[],"mappings":"AAkDA,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAkBjD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gb-title-case.js","sourceRoot":"","sources":["../../../src/tools/gb-title-case.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC5B,MAAM;IACN,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,IAAI;IACJ,OAAO;IACP,KAAK;IACL,KAAK;IACL,IAAI;IACJ,GAAG;IACH,IAAI;CACJ,CAAC,CAAA;AAEF,SAAS,QAAQ,CAAC,IAAY,EAAE,OAAgB;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAEhC,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAErD,0FAA0F;IAC1F,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;QACrB,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAA;QAE5B,iDAAiD;QACjD,IAAI,GAAG,GAAG,CAAC,IAAI,OAAO,KAAK,GAAG;YAAE,OAAO,OAAO,CAAA;QAE9C,8CAA8C;QAC9C,OAAO,OAAO,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,OAAO,KAAK;SACV,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,KAAK;aACjB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;aACxE,IAAI,CAAC,GAAG,CAAC,CAAA;
|
|
1
|
+
{"version":3,"file":"gb-title-case.js","sourceRoot":"","sources":["../../../src/tools/gb-title-case.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC5B,MAAM;IACN,IAAI;IACJ,OAAO;IACP,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,IAAI;IACJ,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,IAAI;IACJ,OAAO;IACP,KAAK;IACL,KAAK;IACL,IAAI;IACJ,GAAG;IACH,IAAI;CACJ,CAAC,CAAA;AAEF,SAAS,QAAQ,CAAC,IAAY,EAAE,OAAgB;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAA;IACtB,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAEhC,IAAI,CAAC,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAA;IAErD,0FAA0F;IAC1F,OAAO,KAAK;SACV,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE;QACrB,IAAI,CAAC,OAAO;YAAE,OAAO,OAAO,CAAA;QAE5B,iDAAiD;QACjD,IAAI,GAAG,GAAG,CAAC,IAAI,OAAO,KAAK,GAAG;YAAE,OAAO,OAAO,CAAA;QAE9C,8CAA8C;QAC9C,OAAO,OAAO,CAAC,CAAC,CAAE,CAAC,WAAW,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IACpD,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,CAAC,CAAA;AACZ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,KAAa;IACxC,KAAK,GAAG,KAAK,CAAC,IAAI,EAAE,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,OAAO,KAAK;SACV,KAAK,CAAC,KAAK,CAAC;SACZ,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACd,MAAM,KAAK,GAAG,KAAK;aACjB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,GAAG,EAAE,SAAS,KAAK,CAAC,IAAI,QAAQ,KAAK,CAAC,CAAC,CAAC;aACxE,IAAI,CAAC,GAAG,CAAC,CAAA;QAEX,SAAS,IAAI,CAAC,CAAA;QAEd,OAAO,KAAK,CAAA;IACb,CAAC,CAAC;SACD,IAAI,CAAC,GAAG,CAAC;SACT,IAAI,EAAE,CAAA;AACT,CAAC"}
|
|
@@ -46,21 +46,37 @@
|
|
|
46
46
|
* - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
|
|
47
47
|
*/
|
|
48
48
|
export interface ExpandGoldenOptions {
|
|
49
|
-
/**
|
|
49
|
+
/**
|
|
50
|
+
* Corpus test shard path(s), comma-separated. Default: the v0.2.0 test shard under the data root.
|
|
51
|
+
*/
|
|
50
52
|
corpus?: string;
|
|
51
|
-
/**
|
|
53
|
+
/**
|
|
54
|
+
* Total seeds to process. Default `100` (pilot).
|
|
55
|
+
*/
|
|
52
56
|
count?: number;
|
|
53
|
-
/**
|
|
57
|
+
/**
|
|
58
|
+
* Variants requested per seed. Default `5`.
|
|
59
|
+
*/
|
|
54
60
|
variants?: number;
|
|
55
|
-
/**
|
|
61
|
+
/**
|
|
62
|
+
* JSONL output path. Default `data/eval/golden/candidates/expand-<ts>.jsonl`.
|
|
63
|
+
*/
|
|
56
64
|
output?: string;
|
|
57
|
-
/**
|
|
65
|
+
/**
|
|
66
|
+
* LLM provider. Default `deepseek`.
|
|
67
|
+
*/
|
|
58
68
|
provider?: "deepseek" | "anthropic";
|
|
59
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* Model id. Default depends on provider.
|
|
71
|
+
*/
|
|
60
72
|
model?: string;
|
|
61
|
-
/**
|
|
73
|
+
/**
|
|
74
|
+
* Parallel LLM calls. Default `4`.
|
|
75
|
+
*/
|
|
62
76
|
concurrency?: number;
|
|
63
|
-
/**
|
|
77
|
+
/**
|
|
78
|
+
* Comma-separated source allow-list.
|
|
79
|
+
*/
|
|
64
80
|
includeSources?: string;
|
|
65
81
|
}
|
|
66
82
|
export interface ExpandGoldenSummary {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"golden-expand.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;
|
|
1
|
+
{"version":3,"file":"golden-expand.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAkDH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,QAAQ,CAAC,EAAE,UAAU,GAAG,WAAW,CAAA;IACnC;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,cAAc,CAAC,EAAE,MAAM,CAAA;CACvB;AAED,MAAM,WAAW,mBAAmB;IACnC,cAAc,EAAE,MAAM,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;CAClB;AAiVD,wBAAsB,YAAY,CACjC,OAAO,GAAE,mBAAwB,EACjC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,mBAAmB,CAAC,CA0F9B"}
|
|
@@ -50,6 +50,11 @@ import { dirname } from "node:path";
|
|
|
50
50
|
import { ParquetReader } from "@dsnp/parquetjs";
|
|
51
51
|
import { $private } from "@mailwoman/core/env";
|
|
52
52
|
import { dataRootPath, writeJSONL } from "@mailwoman/core/utils";
|
|
53
|
+
// ── Types ─────────────────────────────────────────────────────────────────
|
|
54
|
+
/**
|
|
55
|
+
* Longest raw line still plausibly a single address rather than a concatenated record.
|
|
56
|
+
*/
|
|
57
|
+
const MAX_CANDIDATE_LENGTH = 500;
|
|
53
58
|
// ── Seed loading ──────────────────────────────────────────────────────────
|
|
54
59
|
/**
|
|
55
60
|
* Decode BIO labels + tokens into a verified components map. Mirrors the Python `decode_components` in
|
|
@@ -61,7 +66,7 @@ function decodeComponents(tokens, labels) {
|
|
|
61
66
|
let currentTag = null;
|
|
62
67
|
let currentTokens = [];
|
|
63
68
|
const flush = () => {
|
|
64
|
-
if (currentTag && currentTokens.length
|
|
69
|
+
if (currentTag && currentTokens.length && !(currentTag in out)) {
|
|
65
70
|
out[currentTag] = currentTokens.join(" ").trim();
|
|
66
71
|
}
|
|
67
72
|
currentTag = null;
|
|
@@ -145,7 +150,7 @@ async function loadSeeds(corpusPath, count, includeSources, report) {
|
|
|
145
150
|
}
|
|
146
151
|
// Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
|
|
147
152
|
// to sources in alphabetical order. If a pool is smaller than its target, take all of it.
|
|
148
|
-
const sources = Array.from(bySource.keys()).
|
|
153
|
+
const sources = Array.from(bySource.keys()).toSorted();
|
|
149
154
|
const perSource = Math.floor(count / sources.length);
|
|
150
155
|
const remainder = count - perSource * sources.length;
|
|
151
156
|
const picked = [];
|
|
@@ -260,7 +265,7 @@ function makeAnthropicProvider(model) {
|
|
|
260
265
|
}
|
|
261
266
|
function parseCandidates(text) {
|
|
262
267
|
// Strip markdown fences the model sometimes wraps around JSON
|
|
263
|
-
const cleaned = text.
|
|
268
|
+
const cleaned = text.replaceAll(/^```(?:json)?\n?|\n?```$/g, "").trim();
|
|
264
269
|
try {
|
|
265
270
|
const parsed = JSON.parse(cleaned);
|
|
266
271
|
if (Array.isArray(parsed))
|
|
@@ -281,19 +286,21 @@ function parseCandidates(text) {
|
|
|
281
286
|
}
|
|
282
287
|
// ── Validator ─────────────────────────────────────────────────────────────
|
|
283
288
|
function normalize(s) {
|
|
284
|
-
return s.toLowerCase().
|
|
289
|
+
return s.toLowerCase().replaceAll(/\s+/g, " ").trim();
|
|
285
290
|
}
|
|
286
|
-
|
|
291
|
+
/**
|
|
292
|
+
* Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
|
|
293
|
+
*/
|
|
287
294
|
const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"]);
|
|
288
295
|
function validate(seed, candidate) {
|
|
289
296
|
if (!candidate.raw || typeof candidate.raw !== "string")
|
|
290
297
|
return false;
|
|
291
|
-
if (candidate.raw.length >
|
|
298
|
+
if (candidate.raw.length > MAX_CANDIDATE_LENGTH)
|
|
292
299
|
return false;
|
|
293
300
|
if (/```|<\/?\w+>|^\s*\{/.test(candidate.raw))
|
|
294
301
|
return false;
|
|
295
302
|
const normRaw = normalize(candidate.raw);
|
|
296
|
-
const dropped = new Set(candidate.dropped
|
|
303
|
+
const dropped = new Set(candidate.dropped);
|
|
297
304
|
// LLM cannot drop required components present in the seed.
|
|
298
305
|
for (const tag of dropped) {
|
|
299
306
|
if (REQUIRED_COMPONENT_TAGS.has(tag) && seed.components[tag])
|
|
@@ -324,12 +331,12 @@ export async function expandGolden(options = {}, report) {
|
|
|
324
331
|
const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat");
|
|
325
332
|
const concurrencyLimit = options.concurrency ?? 4;
|
|
326
333
|
const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null;
|
|
327
|
-
const ts = new Date().toISOString().
|
|
334
|
+
const ts = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19);
|
|
328
335
|
const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`;
|
|
329
336
|
const provider = providerName === "anthropic" ? makeAnthropicProvider(model) : makeDeepseekProvider(model);
|
|
330
337
|
report?.(`provider: ${provider.name} model: ${provider.model}`);
|
|
331
338
|
const seeds = await loadSeeds(corpusPath, count, includeSources, report);
|
|
332
|
-
if (seeds.length
|
|
339
|
+
if (!seeds.length) {
|
|
333
340
|
throw new Error("no seeds loaded — corpus path or filter is wrong");
|
|
334
341
|
}
|
|
335
342
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
@@ -361,6 +368,7 @@ export async function expandGolden(options = {}, report) {
|
|
|
361
368
|
};
|
|
362
369
|
// Remove dropped components from the components map
|
|
363
370
|
for (const tag of goldenCandidate.dropped_components) {
|
|
371
|
+
// oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record; the object is not on a hot path
|
|
364
372
|
delete goldenCandidate.components[tag];
|
|
365
373
|
}
|
|
366
374
|
outRows.push(goldenCandidate);
|
|
@@ -371,9 +379,9 @@ export async function expandGolden(options = {}, report) {
|
|
|
371
379
|
}
|
|
372
380
|
}
|
|
373
381
|
}
|
|
374
|
-
catch (
|
|
382
|
+
catch (error) {
|
|
375
383
|
errored++;
|
|
376
|
-
report?.(` ✗ seed ${seed.source_id}: ${
|
|
384
|
+
report?.(` ✗ seed ${seed.source_id}: ${error.message}`);
|
|
377
385
|
}
|
|
378
386
|
if ((i + 1) % 10 === 0) {
|
|
379
387
|
report?.(` progress: ${i + 1}/${seeds.length} kept=${kept} dropped=${dropped} errored=${errored}`);
|
|
@@ -387,7 +395,7 @@ export async function expandGolden(options = {}, report) {
|
|
|
387
395
|
report?.(`candidates kept: ${kept}`);
|
|
388
396
|
report?.(`candidates dropped (validator): ${dropped}`);
|
|
389
397
|
report?.(`seeds with errors: ${errored}`);
|
|
390
|
-
report?.(`yield: ${seeds.length
|
|
398
|
+
report?.(`yield: ${seeds.length ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`);
|
|
391
399
|
report?.(`output: ${outputPath}`);
|
|
392
400
|
return { seedsProcessed: seeds.length, kept, dropped, errored, outputPath };
|
|
393
401
|
}
|