@mailwoman/corpus 4.16.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +23 -26
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +25 -28
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +10 -10
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts +2 -2
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +2 -2
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +2 -2
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +9 -9
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +5 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +3 -3
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -10
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +4 -5
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +9 -10
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +13 -14
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +8 -10
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +8 -8
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +6 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +6 -6
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +7 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +9 -9
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +6 -6
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts +2 -2
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +2 -2
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +15 -17
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +6 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -6
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +7 -7
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +12 -12
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +3 -3
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +6 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +17 -19
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +25 -28
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +8 -8
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +11 -11
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +14 -16
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +15 -16
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +13 -15
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +4 -5
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -8
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +6 -8
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +1 -1
- package/out/src/golden.js.map +1 -1
- package/out/src/index.d.ts +1 -0
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/license.d.ts +5 -6
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +5 -6
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +2 -2
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +16 -18
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +22 -23
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +10 -10
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +8 -8
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +16 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -0
- package/out/src/shard-recipes/anchor-absorption.js +73 -0
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts +16 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -0
- package/out/src/shard-recipes/boundary-stress.js +77 -0
- package/out/src/shard-recipes/boundary-stress.js.map +1 -0
- package/out/src/shard-recipes/country-balanced.d.ts +28 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -0
- package/out/src/shard-recipes/country-balanced.js +370 -0
- package/out/src/shard-recipes/country-balanced.js.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts +34 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.js +175 -0
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts +24 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.js +76 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -0
- package/out/src/shard-recipes/fr-order.d.ts +34 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-order.js +238 -0
- package/out/src/shard-recipes/fr-order.js.map +1 -0
- package/out/src/shard-recipes/german.d.ts +19 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -0
- package/out/src/shard-recipes/german.js +170 -0
- package/out/src/shard-recipes/german.js.map +1 -0
- package/out/src/shard-recipes/house-venue.d.ts +14 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/house-venue.js +66 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts +18 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -0
- package/out/src/shard-recipes/index.js +58 -0
- package/out/src/shard-recipes/index.js.map +1 -0
- package/out/src/shard-recipes/intersection.d.ts +38 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -0
- package/out/src/shard-recipes/intersection.js +473 -0
- package/out/src/shard-recipes/intersection.js.map +1 -0
- package/out/src/shard-recipes/locale.d.ts +19 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -0
- package/out/src/shard-recipes/locale.js +257 -0
- package/out/src/shard-recipes/locale.js.map +1 -0
- package/out/src/shard-recipes/no-street.d.ts +14 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -0
- package/out/src/shard-recipes/no-street.js +66 -0
- package/out/src/shard-recipes/no-street.js.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts +34 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.js +686 -0
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -0
- package/out/src/shard-recipes/po-box.d.ts +13 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box.js +93 -0
- package/out/src/shard-recipes/po-box.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +104 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -0
- package/out/src/shard-recipes/scaffold.js +76 -0
- package/out/src/shard-recipes/scaffold.js.map +1 -0
- package/out/src/shard-recipes/street-affix.d.ts +31 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -0
- package/out/src/shard-recipes/street-affix.js +408 -0
- package/out/src/shard-recipes/street-affix.js.map +1 -0
- package/out/src/shard-recipes/street-bare.d.ts +21 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -0
- package/out/src/shard-recipes/street-bare.js +82 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -0
- package/out/src/shard-recipes/street.d.ts +11 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -0
- package/out/src/shard-recipes/street.js +63 -0
- package/out/src/shard-recipes/street.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +25 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -0
- package/out/src/shard-recipes/unit.js +255 -0
- package/out/src/shard-recipes/unit.js.map +1 -0
- package/out/src/split.d.ts +23 -29
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +22 -29
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +2 -3
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +2 -2
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -4
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +3 -4
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -19
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +14 -17
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts +2 -3
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +2 -3
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +2 -2
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +4 -4
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +5 -5
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +27 -23
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -17
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +27 -27
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +8 -9
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +3 -3
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +57 -66
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +56 -65
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +5 -6
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js +5 -6
- package/out/src/tokenize.js.map +1 -1
- package/out/src/types.d.ts +49 -51
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +32 -34
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +23 -25
- package/out/src/wof-json.js.map +1 -1
- package/package.json +14 -14
package/out/src/license.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH
|
|
1
|
+
{"version":3,"file":"license.js","sourceRoot":"","sources":["../../src/license.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH;;;GAGG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,gDAAgD,CAAA;AAEnF;;;;;GAKG;AACH,MAAM,UAAU,sBAAsB,CAAC,IAAY;IAClD,OAAO,IAAI;SACT,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC;SACf,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,qBAAqB,EAAE,MAAM,CAAC,EAAE,GAAG,CAAC,CAAC,CAAA;AAC9E,CAAC;AAED,kGAAkG;AAClG,MAAM,UAAU,eAAe,CAAC,OAA2B,EAAE,QAA2B;IACvF,MAAM,CAAC,GAAG,OAAO,IAAI,EAAE,CAAA;IAEvB,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;
|
|
1
|
+
{"version":3,"file":"reader.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,0CAA0C,CAAA;AACnF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAAE,KAAK,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAEnE,gEAAgE;AAChE,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;WAEV,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,QAAQ,EAAE,MAAM,GAAG,GAAG,EACtB,OAAO,CAAC,EAAE,mBAAmB,GAC3B,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;WAMN,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,mBAAmB;WAMrF,kBAAkB,CAAC,CAAC,SAAS,iBAAiB,EACnE,cAAc,EAAE,qBAAqB,EACrC,IAAI,CAAC,EAAE,mBAAmB;IAkBX,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC;IAI7D,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;
|
|
1
|
+
{"version":3,"file":"reader.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/reader.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAA0B,aAAa,EAAE,MAAM,aAAa,CAAA;AAEnE,gEAAgE;AAChE,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAGhF,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,QAAsB,EACtB,OAA6B;QAE7B,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAA;QAEzF,OAAO,aAAa,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,UAAU,CAA8B,MAAc,EAAE,OAA6B;QAC1G,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QAE9E,OAAO,IAAI,CAAC,kBAAkB,CAAI,cAAc,EAAE,OAAO,CAAC,CAAA;IAC3D,CAAC;IAED,MAAM,CAAU,KAAK,CAAC,kBAAkB,CACvC,cAAqC,EACrC,IAA0B;QAE1B,IAAI,IAAI,EAAE,QAAQ,EAAE,CAAC;YACpB,OAAO,IAAI,aAAa,CAAI,IAAI,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QACjE,CAAC;QAED,IAAI,CAAC;YACJ,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAEjC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,UAAU,EAAE,CAAA;YAElD,OAAO,IAAI,aAAa,CAAI,QAAQ,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;QAC5D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,cAAc,CAAC,KAAK,EAAE,CAAA;YAC5B,MAAM,GAAG,CAAA;QACV,CAAC;IACF,CAAC;IAEe,CAAC,MAAM,CAAC,aAAa,CAAC;QACrC,OAAO,KAAK,CAAC,MAAM,CAAC,aAAa,CAAC,EAAsC,CAAA;IACzE,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC;CACD"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;
|
|
1
|
+
{"version":3,"file":"writer.d.ts","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAKH,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AACpE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qCAAqC,CAAA;AACxE,OAAO,EAAU,KAAK,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAGlF,OAAO,EACN,KAAK,iBAAiB,EACtB,aAAa,EACb,KAAK,uBAAuB,EAC5B,4BAA4B,EAC5B,MAAM,aAAa,CAAA;AAEpB,gEAAgE;AAChE,qBAAa,aAAa,CAAC,CAAC,SAAS,iBAAiB,CAAE,SAAQ,iBAAkB,YAAW,eAAe;;IACnG,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,+BAAqC;WAG9D,UAAU,CAAC,CAAC,SAAS,iBAAiB,EAC3D,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,YAAY,EAAE,kBAAkB,EAChC,IAAI,GAAE,aAAkB,GACtB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B,oGAAoG;WAC9E,QAAQ,CAAC,CAAC,SAAS,iBAAiB,EACzD,UAAU,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,uBAAuB,CAAC,CAAC,CAAC,EACzD,UAAU,EAAE,MAAM,GAAG,MAAM,GAAG,GAAG,EACjC,IAAI,CAAC,EAAE,aAAa,GAClB,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC;IAW5B,mDAAmD;IACnC,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAI7D,2FAA2F;IACrE,SAAS,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAItD,8EAA8E;IACxD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAc/B,CAAC,MAAM,CAAC,YAAY,CAAC;IAIrB,OAAO;CAGpB"}
|
|
@@ -12,11 +12,11 @@
|
|
|
12
12
|
* and closes cleanly. `close()` internally serializes against any in-flight flush so back-to-back
|
|
13
13
|
* dispose calls don't race.
|
|
14
14
|
*/
|
|
15
|
+
import * as fs from "node:fs/promises";
|
|
16
|
+
import * as path from "node:path";
|
|
15
17
|
import { ParquetWriter as BaseParquetWriter } from "@dsnp/parquetjs";
|
|
16
18
|
import { osopen } from "@dsnp/parquetjs/dist/lib/util.js";
|
|
17
19
|
import { ParquetEnvelopeWriter } from "@dsnp/parquetjs/dist/lib/writer.js";
|
|
18
|
-
import * as fs from "node:fs/promises";
|
|
19
|
-
import * as path from "node:path";
|
|
20
20
|
import { ParquetSchema, ParquetSchemaDefinitionCache, } from "./schema.js";
|
|
21
21
|
/** A typed Parquet writer, wrapping the base Parquet writer. */
|
|
22
22
|
export class ParquetWriter extends BaseParquetWriter {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,
|
|
1
|
+
{"version":3,"file":"writer.js","sourceRoot":"","sources":["../../../src/parquet-wrapper/writer.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACtC,OAAO,KAAK,IAAI,MAAM,WAAW,CAAA;AAEjC,OAAO,EAAE,aAAa,IAAI,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEpE,OAAO,EAAE,MAAM,EAA2B,MAAM,kCAAkC,CAAA;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAA;AAE1E,OAAO,EAEN,aAAa,EAEb,4BAA4B,GAC5B,MAAM,aAAa,CAAA;AAEpB,gEAAgE;AAChE,MAAM,OAAO,aAA2C,SAAQ,iBAAiB;IAEtE,MAAM,CAAU,qBAAqB,GAAG,IAAI,4BAA4B,EAAE,CAAA;IACpF,SAAS,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAA;IAE5C,MAAM,CAAU,KAAK,CAAC,UAAU,CAC/B,UAAyD,EACzD,YAAgC,EAChC,OAAsB,EAAE;QAExB,MAAM,MAAM,GACX,UAAU,YAAY,aAAa;YAClC,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,aAAa,CAAC,qBAAqB,CAAC,kBAAkB,CAAC,UAAU,CAAC,CAAA;QAEtE,MAAM,cAAc,GAAG,MAAM,qBAAqB,CAAC,UAAU,CAAC,MAAM,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;QAEzF,OAAO,IAAI,aAAa,CAAI,MAAM,EAAE,cAAc,EAAE,IAAI,CAAC,CAAA;IAC1D,CAAC;IAED,oGAAoG;IACpG,MAAM,CAAU,KAAK,CAAC,QAAQ,CAC7B,UAAyD,EACzD,UAAiC,EACjC,IAAoB;QAEpB,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,MAAM,YAAY,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,CAAA;QAEnD,OAAO,aAAa,CAAC,UAAU,CAAI,UAAU,EAAE,YAAY,EAAE,IAAI,CAAC,CAAA;IACnE,CAAC;IAED,6DAA6D;IAC7D,mDAAmD;IACnC,WAAW,CAAC,GAAW,EAAE,KAAa;QACrD,OAAO,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IACrC,CAAC;IAED,2FAA2F;IAC3E,KAAK,CAAC,SAAS,CAAC,GAAM;QACrC,OAAO,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED,8EAA8E;IAC9D,KAAK,CAAC,KAAK;QAC1B,MAAM,IAAI,CAAC,SAAS,CAAA;QAEpB,IAAI,IAAI,CAAC,MAAM;YAAE,OAAM;QAEvB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,aAAa,EAAQ,CAAA;QAElE,KAAK,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QAEnC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAA;QAExB,OAAO,IAAI,CAAC,SAAS,CAAA;IACtB,CAAC;IAEM,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QACjC,OAAO,IAAI,CAAC,KAAK,EAAE,CAAA;IACpB,CAAC;IAEM,KAAK,CAAC,OAAO;QACnB,OAAO,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAA;IACnC,CAAC"}
|
package/out/src/parquet.d.ts
CHANGED
|
@@ -44,9 +44,8 @@ export declare const ROW_GROUP_SIZE = 50000;
|
|
|
44
44
|
/** Snappy is the only zstd-equivalent codec available in @dsnp/parquetjs 1.7.0. */
|
|
45
45
|
export declare const SHARD_COMPRESSION: "SNAPPY";
|
|
46
46
|
/**
|
|
47
|
-
* A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for
|
|
48
|
-
*
|
|
49
|
-
* rows.
|
|
47
|
+
* A single Parquet-style row shape. The `[key: string]: unknown` index signature is required for compatibility with
|
|
48
|
+
* `ParquetRecordLike` in the wrapper — parquetjs accepts any string key on rows.
|
|
50
49
|
*/
|
|
51
50
|
export interface ParquetRow {
|
|
52
51
|
raw: string;
|
|
@@ -68,8 +67,8 @@ export interface ParquetRow {
|
|
|
68
67
|
/** Column names emitted into every shard. Matches `ParquetRow`. */
|
|
69
68
|
export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
|
|
70
69
|
/**
|
|
71
|
-
* Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8
|
|
72
|
-
*
|
|
70
|
+
* Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
|
|
71
|
+
* tokens/labels arrays. Compression is per-column SNAPPY.
|
|
73
72
|
*/
|
|
74
73
|
export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
|
|
75
74
|
/** Per-shard metadata captured in `MANIFEST.json`. */
|
|
@@ -102,9 +101,9 @@ export interface WriteShardsOptions {
|
|
|
102
101
|
rowsPerShard?: number;
|
|
103
102
|
}
|
|
104
103
|
/**
|
|
105
|
-
* Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's
|
|
106
|
-
*
|
|
107
|
-
*
|
|
104
|
+
* Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
|
|
105
|
+
* time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
|
|
106
|
+
* lookup table.
|
|
108
107
|
*
|
|
109
108
|
* Splits with no rows can be omitted (or passed as an empty iterable); `writeShards` skips them.
|
|
110
109
|
*/
|
|
@@ -112,20 +111,19 @@ export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>
|
|
|
112
111
|
/**
|
|
113
112
|
* Project a labeled row to the Parquet schema.
|
|
114
113
|
*
|
|
115
|
-
* The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
* failure, naming the row, instead.
|
|
114
|
+
* The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
|
|
115
|
+
* from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the shard (the "builders
|
|
116
|
+
* before parquet = silent loss" hazard). Loud failure, naming the row, instead.
|
|
119
117
|
*/
|
|
120
118
|
export declare function rowToParquet(row: LabeledRow): ParquetRow;
|
|
121
119
|
/**
|
|
122
|
-
* Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed
|
|
123
|
-
*
|
|
124
|
-
*
|
|
120
|
+
* Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed sequentially so that
|
|
121
|
+
* only one shard writer is open at a time — memory cost is bounded by the parquetjs row-group buffer (~`ROW_GROUP_SIZE
|
|
122
|
+
* × row_size`), not by the labeled-row count.
|
|
125
123
|
*
|
|
126
|
-
* Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior
|
|
127
|
-
*
|
|
128
|
-
*
|
|
124
|
+
* Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
|
|
125
|
+
* because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
|
|
126
|
+
* for the new wire-up.
|
|
129
127
|
*/
|
|
130
128
|
export declare function writeShards(perSplit: PerSplitRows, opts: WriteShardsOptions): Promise<ShardManifest>;
|
|
131
129
|
//# sourceMappingURL=parquet.d.ts.map
|
package/out/src/parquet.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;
|
|
1
|
+
{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAOH,OAAO,EAAiB,KAAK,uBAAuB,EAAE,MAAM,4BAA4B,CAAA;AACxF,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAA;AAC3C,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE5C,sFAAsF;AACtF,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC,mFAAmF;AACnF,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD;;;GAGG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,EAAE,MAAM,GAAG,IAAI,CAAA;IACrB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,EAAE,MAAM,GAAG,IAAI,CAAA;IAC3B,aAAa,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED,mEAAmE;AACnE,eAAO,MAAM,eAAe,yLAelB,CAAA;AAEV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;CACtB;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC,oEAAoE;IACpE,SAAS,EAAE,MAAM,CAAA;IAEjB,yEAAyE;IACzE,aAAa,EAAE,MAAM,CAAA;IAErB,6EAA6E;IAC7E,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAqCxD;AA+BD;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CA4F1G"}
|
package/out/src/parquet.js
CHANGED
|
@@ -63,8 +63,8 @@ export const PARQUET_COLUMNS = [
|
|
|
63
63
|
"synth_base_id",
|
|
64
64
|
];
|
|
65
65
|
/**
|
|
66
|
-
* Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8
|
|
67
|
-
*
|
|
66
|
+
* Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
|
|
67
|
+
* tokens/labels arrays. Compression is per-column SNAPPY.
|
|
68
68
|
*/
|
|
69
69
|
export const LABELED_ROW_SCHEMA = {
|
|
70
70
|
raw: { type: "UTF8", compression: SHARD_COMPRESSION },
|
|
@@ -88,10 +88,9 @@ export const LABELED_ROW_SCHEMA = {
|
|
|
88
88
|
/**
|
|
89
89
|
* Project a labeled row to the Parquet schema.
|
|
90
90
|
*
|
|
91
|
-
* The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
* failure, naming the row, instead.
|
|
91
|
+
* The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
|
|
92
|
+
* from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the shard (the "builders
|
|
93
|
+
* before parquet = silent loss" hazard). Loud failure, naming the row, instead.
|
|
95
94
|
*/
|
|
96
95
|
export function rowToParquet(row) {
|
|
97
96
|
const { span_starts, span_ends, span_tags } = row;
|
|
@@ -125,9 +124,9 @@ export function rowToParquet(row) {
|
|
|
125
124
|
};
|
|
126
125
|
}
|
|
127
126
|
/**
|
|
128
|
-
* Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns;
|
|
129
|
-
*
|
|
130
|
-
*
|
|
127
|
+
* Project a `ParquetRow` for `appendRow`. parquetjs treats `null` as "skip" for `optional` columns; passing it
|
|
128
|
+
* explicitly is fine, but cleaner to omit so the on-disk Definition Levels match what PyArrow / DuckDB / etc. produce
|
|
129
|
+
* for the same logical row.
|
|
131
130
|
*/
|
|
132
131
|
function appendShape(row) {
|
|
133
132
|
const out = {
|
|
@@ -152,13 +151,13 @@ function appendShape(row) {
|
|
|
152
151
|
return out;
|
|
153
152
|
}
|
|
154
153
|
/**
|
|
155
|
-
* Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed
|
|
156
|
-
*
|
|
157
|
-
*
|
|
154
|
+
* Stream labeled rows into `.parquet` shards, one set of shards per split. Splits are processed sequentially so that
|
|
155
|
+
* only one shard writer is open at a time — memory cost is bounded by the parquetjs row-group buffer (~`ROW_GROUP_SIZE
|
|
156
|
+
* × row_size`), not by the labeled-row count.
|
|
158
157
|
*
|
|
159
|
-
* Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior
|
|
160
|
-
*
|
|
161
|
-
*
|
|
158
|
+
* Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
|
|
159
|
+
* because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
|
|
160
|
+
* for the new wire-up.
|
|
162
161
|
*/
|
|
163
162
|
export async function writeShards(perSplit, opts) {
|
|
164
163
|
const rowsPerShard = opts.rowsPerShard ?? 1_000_000;
|
|
@@ -175,8 +174,8 @@ export async function writeShards(perSplit, opts) {
|
|
|
175
174
|
let writer = null;
|
|
176
175
|
let path = "";
|
|
177
176
|
let shardRows = 0;
|
|
178
|
-
let
|
|
179
|
-
let
|
|
177
|
+
let firstSourceID = "";
|
|
178
|
+
let lastSourceID = "";
|
|
180
179
|
const openShard = async () => {
|
|
181
180
|
const splitDir = join(corpusDir, split);
|
|
182
181
|
await mkdir(splitDir, { recursive: true });
|
|
@@ -188,8 +187,8 @@ export async function writeShards(perSplit, opts) {
|
|
|
188
187
|
writer.setMetadata("mailwoman.split", split);
|
|
189
188
|
writer.setMetadata("mailwoman.shard_index", String(shardIndex));
|
|
190
189
|
shardRows = 0;
|
|
191
|
-
|
|
192
|
-
|
|
190
|
+
firstSourceID = "";
|
|
191
|
+
lastSourceID = "";
|
|
193
192
|
};
|
|
194
193
|
const closeShard = async () => {
|
|
195
194
|
if (!writer)
|
|
@@ -206,8 +205,8 @@ export async function writeShards(perSplit, opts) {
|
|
|
206
205
|
rows: shardRows,
|
|
207
206
|
bytes: fileStat.size,
|
|
208
207
|
sha256,
|
|
209
|
-
first_source_id:
|
|
210
|
-
last_source_id:
|
|
208
|
+
first_source_id: firstSourceID,
|
|
209
|
+
last_source_id: lastSourceID,
|
|
211
210
|
});
|
|
212
211
|
}
|
|
213
212
|
writer = null;
|
|
@@ -218,8 +217,8 @@ export async function writeShards(perSplit, opts) {
|
|
|
218
217
|
const pq = rowToParquet(row);
|
|
219
218
|
await writer.appendRow(appendShape(pq));
|
|
220
219
|
if (shardRows === 0)
|
|
221
|
-
|
|
222
|
-
|
|
220
|
+
firstSourceID = row.source_id;
|
|
221
|
+
lastSourceID = row.source_id;
|
|
223
222
|
shardRows++;
|
|
224
223
|
counts[split]++;
|
|
225
224
|
totalRows++;
|
package/out/src/parquet.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../src/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAqCG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAA;AAC1C,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACzD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,aAAa,EAAgC,MAAM,4BAA4B,CAAA;AAIxF,sFAAsF;AACtF,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC,mFAAmF;AACnF,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAwBlD,mEAAmE;AACnE,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,wFAAwF;IACxF,+FAA+F;IAC/F,gFAAgF;IAChF,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC5E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AA6CD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,GAAe;IACnC,MAAM,GAAG,GAA4B;QACpC,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;KACpB,CAAA;IAED,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI;QAAE,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,CAAA;IAEhD,IAAI,GAAG,CAAC,YAAY,KAAK,IAAI;QAAE,GAAG,CAAC,YAAY,GAAG,GAAG,CAAC,YAAY,CAAA;IAElE,IAAI,GAAG,CAAC,aAAa,KAAK,IAAI;QAAE,GAAG,CAAC,aAAa,GAAG,GAAG,CAAC,aAAa,CAAA;IAErE,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,MAAM,GAAqC,IAAI,CAAA;QACnD,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YAC1C,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAC5E,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAa,kBAAkB,EAAE,IAAI,EAAE;gBAC3E,YAAY,EAAE,cAAc;aAC5B,CAAC,CAAA;YACF,MAAM,CAAC,WAAW,CAAC,0BAA0B,EAAE,IAAI,CAAC,aAAa,CAAC,CAAA;YAClE,MAAM,CAAC,WAAW,CAAC,iBAAiB,EAAE,KAAK,CAAC,CAAA;YAC5C,MAAM,CAAC,WAAW,CAAC,uBAAuB,EAAE,MAAM,CAAC,UAAU,CAAC,CAAC,CAAA;YAC/D,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,IAAI,CAAC,MAAM;gBAAE,OAAM;YACnB,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YAEpB,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,CAAA;gBACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;gBACnC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,CAAC,IAAI;oBACpB,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YACD,MAAM,GAAG,IAAI,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,MAAM;gBAAE,MAAM,SAAS,EAAE,CAAA;YAC9B,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAC5B,MAAM,MAAO,CAAC,SAAS,CAAC,WAAW,CAAC,EAAE,CAA0B,CAAC,CAAA;YAEjE,IAAI,SAAS,KAAK,CAAC;gBAAE,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAClD,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAC5B,SAAS,EAAE,CAAA;YACX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YACf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAClB,UAAU,EAAE,CAAA;YACb,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAEnG,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,+FAA+F;AAC/F,KAAK,UAAU,QAAQ,CAAC,IAAY;IACnC,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAA;IACjC,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;IAErC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM;QAAE,IAAI,CAAC,MAAM,CAAC,KAAe,CAAC,CAAA;IAE9D,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC1B,CAAC"}
|
package/out/src/runner.d.ts
CHANGED
|
@@ -31,7 +31,7 @@ import type { AdapterOptions, CorpusAdapter } from "./types.js";
|
|
|
31
31
|
/** Snapshot of the runner's state, emitted on every progress tick. */
|
|
32
32
|
export interface RunnerProgress {
|
|
33
33
|
/** Adapter being driven. */
|
|
34
|
-
|
|
34
|
+
adapterID: string;
|
|
35
35
|
/** Total rows the adapter has yielded (before dedup). */
|
|
36
36
|
yielded: number;
|
|
37
37
|
/** Rows actually written to JSONL (after dedup). */
|
|
@@ -52,13 +52,13 @@ export interface RunAdapterOptions {
|
|
|
52
52
|
/** Corpus version stamped onto every row. Locked together with the tokenizer version. */
|
|
53
53
|
corpusVersion: string;
|
|
54
54
|
/**
|
|
55
|
-
* Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once
|
|
56
|
-
*
|
|
55
|
+
* Optional progress callback. Invoked every `progressEvery` rows yielded (default 1000) and once at the end of the
|
|
56
|
+
* run. Errors thrown from this callback abort the run.
|
|
57
57
|
*/
|
|
58
58
|
onProgress?: (snapshot: RunnerProgress) => void;
|
|
59
59
|
/**
|
|
60
|
-
* Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always
|
|
61
|
-
*
|
|
60
|
+
* Yielded-row interval at which `onProgress` fires. Defaults to 1000. The terminal tick is always emitted regardless
|
|
61
|
+
* of this value.
|
|
62
62
|
*/
|
|
63
63
|
progressEvery?: number;
|
|
64
64
|
}
|
|
@@ -81,14 +81,14 @@ export interface AdapterRunManifest {
|
|
|
81
81
|
/**
|
|
82
82
|
* Drive a single adapter to completion.
|
|
83
83
|
*
|
|
84
|
-
* Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under
|
|
85
|
-
*
|
|
86
|
-
*
|
|
84
|
+
* Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
|
|
85
|
+
* Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
|
|
86
|
+
* signal fires.
|
|
87
87
|
*/
|
|
88
88
|
export declare function runAdapter(opts: RunAdapterOptions): Promise<AdapterRunManifest>;
|
|
89
89
|
/**
|
|
90
|
-
* Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the
|
|
91
|
-
*
|
|
90
|
+
* Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
|
|
91
|
+
* calling if partial-failure is desired).
|
|
92
92
|
*
|
|
93
93
|
* Returns the manifests in registry insertion order.
|
|
94
94
|
*/
|
package/out/src/runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAMH,OAAO,EAAsC,KAAK,eAAe,EAAwB,MAAM,cAAc,CAAA;AAC7G,OAAO,KAAK,EAAE,cAAc,EAAgB,aAAa,EAAE,MAAM,YAAY,CAAA;AAE7E,sEAAsE;AACtE,MAAM,WAAW,cAAc;IAC9B,4BAA4B;IAC5B,SAAS,EAAE,MAAM,CAAA;IAEjB,yDAAyD;IACzD,OAAO,EAAE,MAAM,CAAA;IAEf,oDAAoD;IACpD,OAAO,EAAE,MAAM,CAAA;IAEf,qCAAqC;IACrC,KAAK,EAAE,MAAM,CAAA;IAEb,qDAAqD;IACrD,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,+CAA+C;AAC/C,MAAM,WAAW,iBAAiB;IACjC,wBAAwB;IACxB,OAAO,EAAE,aAAa,CAAA;IAEtB,iFAAiF;IACjF,cAAc,EAAE,cAAc,CAAA;IAE9B,sFAAsF;IACtF,SAAS,EAAE,MAAM,CAAA;IAEjB,yFAAyF;IACzF,aAAa,EAAE,MAAM,CAAA;IAErB;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,cAAc,KAAK,IAAI,CAAA;IAE/C;;;OAGG;IACH,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAED,+EAA+E;AAC/E,MAAM,WAAW,kBAAkB;IAClC,UAAU,EAAE,MAAM,CAAA;IAClB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,MAAM,CAAA;IACvB,WAAW,EAAE,MAAM,CAAA;IACnB,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;IACf,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,UAAU,EAAE,MAAM,CAAA;IAClB,UAAU,EAAE,MAAM,CAAA;IAClB,QAAQ,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;CAClB;AAED;;;;;;GAMG;AACH,wBAAsB,UAAU,CAAC,IAAI,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAoGrF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,IAAI,CAAC,iBAAiB,EAAE,SAAS,CAAC,GAAG;IAAE,iBAAiB,CAAC,EAAE,CAAC,CAAC,EAAE,aAAa,KAAK,cAAc,CAAA;CAAE,GACvG,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAe/B;AA4CD,qEAAqE;AACrE,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAErE"}
|
package/out/src/runner.js
CHANGED
|
@@ -33,9 +33,9 @@ import { canonicalDedupKey, streamingSha256 } from "./adapter.js";
|
|
|
33
33
|
/**
|
|
34
34
|
* Drive a single adapter to completion.
|
|
35
35
|
*
|
|
36
|
-
* Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under
|
|
37
|
-
*
|
|
38
|
-
*
|
|
36
|
+
* Returns the manifest describing the run. Writes `canonical.jsonl` + `MANIFEST.json` under `outputDir/<adapter.id>/`.
|
|
37
|
+
* Throws if the output directory cannot be created, if a row arrives with a missing required field, or if the abort
|
|
38
|
+
* signal fires.
|
|
39
39
|
*/
|
|
40
40
|
export async function runAdapter(opts) {
|
|
41
41
|
const { adapter, adapterOptions, outputDir, corpusVersion } = opts;
|
|
@@ -56,7 +56,7 @@ export async function runAdapter(opts) {
|
|
|
56
56
|
let bytes = 0;
|
|
57
57
|
const emitProgress = () => {
|
|
58
58
|
opts.onProgress?.({
|
|
59
|
-
|
|
59
|
+
adapterID: adapter.id,
|
|
60
60
|
yielded,
|
|
61
61
|
written,
|
|
62
62
|
bytes,
|
|
@@ -123,8 +123,8 @@ export async function runAdapter(opts) {
|
|
|
123
123
|
return manifest;
|
|
124
124
|
}
|
|
125
125
|
/**
|
|
126
|
-
* Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the
|
|
127
|
-
*
|
|
126
|
+
* Drive every adapter in a registry sequentially. Stops on the first failure (caller can filter the registry before
|
|
127
|
+
* calling if partial-failure is desired).
|
|
128
128
|
*
|
|
129
129
|
* Returns the manifests in registry insertion order.
|
|
130
130
|
*/
|
|
@@ -141,8 +141,8 @@ export async function runAllAdapters(registry, common) {
|
|
|
141
141
|
return out;
|
|
142
142
|
}
|
|
143
143
|
/**
|
|
144
|
-
* Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL
|
|
145
|
-
*
|
|
144
|
+
* Validate an emitted row. Cheap; runs once per row. Catches adapter bugs early so the JSONL doesn't end up
|
|
145
|
+
* half-malformed.
|
|
146
146
|
*/
|
|
147
147
|
function assertEmittedRow(adapter, row) {
|
|
148
148
|
if (row.source !== adapter.id) {
|
package/out/src/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAEH,OAAO,EAAE,iBAAiB,EAAoB,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AACnD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEzC,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAA8C,MAAM,cAAc,CAAA;AAiE7G;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,IAAuB;IACvD,MAAM,EAAE,OAAO,EAAE,cAAc,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,IAAI,CAAA;IAClE,MAAM,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,KAAK,CAAA;IAEjD,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,EAAE,CAAC,CAAA;IAC9C,MAAM,KAAK,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE5C,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAA;IACrD,MAAM,YAAY,GAAG,IAAI,CAAC,UAAU,EAAE,eAAe,CAAC,CAAA;IAEtD,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAA;IAC5B,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;IAE5B,MAAM,MAAM,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,MAAM,GAAoB,eAAe,EAAE,CAAA;IACjD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAC9B,MAAM,cAAc,GAAG,UAAU,CAAA;IACjC,IAAI,cAAc,GAAG,KAAK,CAAA;IAE1B,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,MAAM,YAAY,GAAG,GAAS,EAAE;QAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,SAAS,EAAE,OAAO,CAAC,EAAE;YACrB,OAAO;YACP,OAAO;YACP,KAAK;YACL,UAAU,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE;SAClC,CAAC,CAAA;IACH,CAAC,CAAA;IAED,IAAI,CAAC;QACJ,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,EAAE,CAAC;YACtD,IAAI,cAAc,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;gBACpC,MAAM,IAAI,YAAY,CAAC,+BAA+B,EAAE,YAAY,CAAC,CAAA;YACtE,CAAC;YAED,OAAO,EAAE,CAAA;YACT,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,CAAA;YAE9B,MAAM,OAAO,GAAiB,EAAE,GAAG,GAAG,EAAE,cAAc,EAAE,aAAa,EAAE,CAAA;YACvE,MAAM,GAAG,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAA;YAEtC,IAAI,CAAC,cAAc,EAAE,CAAC;gBACrB,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACnB,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC;wBAAE,YAAY,EAAE,CAAA;oBACjD,SAAQ;gBACT,CAAC;gBAED,IAAI,IAAI,CAAC,IAAI,IAAI,cAAc,EAAE,CAAC;oBACjC,cAAc,GAAG,IAAI,CAAA;oBACrB,OAAO,CAAC,MAAM,CAAC,KAAK,CACnB,+BAA+B,cAAc,CAAC,cAAc,EAAE,wCAAwC,CACtG,CAAA;gBACF,CAAC;qBAAM,CAAC;oBACP,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;gBACd,CAAC;YACF,CAAC;YAED,MAAM,IAAI,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAA;YAC3C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACnB,KAAK,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;YACxC,OAAO,EAAE,CAAA;YAET,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;YAC5B,CAAC;YAED,IAAI,OAAO,GAAG,aAAa,KAAK,CAAC;gBAAE,YAAY,EAAE,CAAA;QAClD,CAAC;IACF,CAAC;YAAS,CAAC;QACV,MAAM,CAAC,GAAG,EAAE,CAAA;QACZ,MAAM,IAAI,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;IAC5B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAA;IAC1B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,CAAA;IACzC,YAAY,EAAE,CAAA;IAEd,MAAM,QAAQ,GAAuB;QACpC,UAAU,EAAE,OAAO,CAAC,EAAE;QACtB,cAAc,EAAE,aAAa;QAC7B,eAAe,EAAE,OAAO,CAAC,cAAc;QACvC,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,OAAO;QACP,OAAO;QACP,OAAO,EAAE,OAAO,GAAG,OAAO;QAC1B,KAAK;QACL,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE;QACvB,UAAU,EAAE,SAAS;QACrB,UAAU,EAAE,SAAS,CAAC,WAAW,EAAE;QACnC,QAAQ,EAAE,OAAO,CAAC,WAAW,EAAE;QAC/B,UAAU;KACV,CAAA;IAED,MAAM,SAAS,CAAC,YAAY,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE/E,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAAyB,EACzB,MAAyG;IAEzG,MAAM,GAAG,GAAyB,EAAE,CAAA;IAEpC,KAAK,MAAM,OAAO,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,MAAM,cAAc,GAAG,MAAM,CAAC,iBAAiB,EAAE,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,cAAc,CAAA;QACnF,GAAG,CAAC,IAAI,CACP,MAAM,UAAU,CAAC;YAChB,GAAG,MAAM;YACT,OAAO;YACP,cAAc;SACd,CAAC,CACF,CAAA;IACF,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,OAAsB,EAAE,GAAiB;IAClE,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,2CAA2C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IAC/G,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,0BAA0B,CAAC,CAAA;IACjE,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,oCAAoC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC1F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CAAC,WAAW,OAAO,CAAC,EAAE,wCAAwC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAA;IAC9F,CAAC;AACF,CAAC;AAED,gGAAgG;AAChG,SAAS,IAAI,CAAC,OAAoB,EAAE,KAAwB;IAC3D,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACtC,MAAM,OAAO,GAAG,GAAS,EAAE;YAC1B,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,OAAO,EAAE,CAAA;QACV,CAAC,CAAA;QACD,MAAM,OAAO,GAAG,CAAC,GAAU,EAAQ,EAAE;YACpC,OAAO,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;YAC3B,MAAM,CAAC,GAAG,CAAC,CAAA;QACZ,CAAC,CAAA;QACD,OAAO,CAAC,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAC5B,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;IAC/B,CAAC,CAAC,CAAA;AACH,CAAC;AAED,qEAAqE;AACrE,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACrD,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;AACpD,CAAC"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `anchor-absorption` shard recipe (#220/#723, Probe A1) — self-generates `--count` rows from
|
|
7
|
+
* {@link synthesizeAnchorAbsorptionRow}'s 6-slice mix (CASE-H / CASE-P-us-rural / CASE-P-de /
|
|
8
|
+
* anchor-fp / locale-ambig / standard), aligns each to BIO, and emits a labeled JSONL. The
|
|
9
|
+
* LEADING 5-digit on CASE-H/anchor-fp/locale-ambig is sampled from the REAL US ZIPs in the
|
|
10
|
+
* postcode-anchor lookup, so the shaped-painted anchor fires on it exactly as inference does —
|
|
11
|
+
* teaching the model to OVERRIDE a present anchor from context. Ported from
|
|
12
|
+
* scripts/build-anchor-absorption-shard.mjs.
|
|
13
|
+
*/
|
|
14
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
15
|
+
export declare const anchorAbsorptionRecipe: ShardRecipe;
|
|
16
|
+
//# sourceMappingURL=anchor-absorption.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anchor-absorption.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AASH,OAAO,EAAW,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AAoBzD,eAAO,MAAM,sBAAsB,EAAE,WA4CpC,CAAA"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `anchor-absorption` shard recipe (#220/#723, Probe A1) — self-generates `--count` rows from
|
|
7
|
+
* {@link synthesizeAnchorAbsorptionRow}'s 6-slice mix (CASE-H / CASE-P-us-rural / CASE-P-de /
|
|
8
|
+
* anchor-fp / locale-ambig / standard), aligns each to BIO, and emits a labeled JSONL. The
|
|
9
|
+
* LEADING 5-digit on CASE-H/anchor-fp/locale-ambig is sampled from the REAL US ZIPs in the
|
|
10
|
+
* postcode-anchor lookup, so the shaped-painted anchor fires on it exactly as inference does —
|
|
11
|
+
* teaching the model to OVERRIDE a present anchor from context. Ported from
|
|
12
|
+
* scripts/build-anchor-absorption-shard.mjs.
|
|
13
|
+
*/
|
|
14
|
+
import { readFileSync } from "node:fs";
|
|
15
|
+
import { dataRootPath } from "@mailwoman/core/utils";
|
|
16
|
+
import { stableSourceID } from "../adapter.js";
|
|
17
|
+
import { alignRow } from "../align.js";
|
|
18
|
+
import { synthesizeAnchorAbsorptionRow } from "../synthesize-anchor-absorption.js";
|
|
19
|
+
import { makeLcg } from "./scaffold.js";
|
|
20
|
+
// The leading-5-digit source: the real US ZIPs in the postcode anchor's pilot lookup. Resolved through
|
|
21
|
+
// the data-root helper (the lab default is `$MAILWOMAN_DATA_ROOT`), never re-hardcoded.
|
|
22
|
+
const ANCHOR_LOOKUP = dataRootPath("anchor", "pilot-anchor-lookup.json");
|
|
23
|
+
/** The real US ZIPs in the anchor lookup (entries whose value is a `[{ US: … }]` candidate list). */
|
|
24
|
+
function loadRealUsZips(path) {
|
|
25
|
+
const d = JSON.parse(readFileSync(path, "utf8"));
|
|
26
|
+
const zips = [];
|
|
27
|
+
for (const [pc, v] of Object.entries(d)) {
|
|
28
|
+
if (Array.isArray(v) && v[0] && typeof v[0] === "object" && "US" in v[0] && /^\d{5}$/.test(pc)) {
|
|
29
|
+
zips.push(pc);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return zips;
|
|
33
|
+
}
|
|
34
|
+
export const anchorAbsorptionRecipe = {
|
|
35
|
+
name: "anchor-absorption",
|
|
36
|
+
description: "Anchor-absorption counter-augmentation (#220/#723): 6-slice mix → synthesizeAnchorAbsorptionRow",
|
|
37
|
+
mode: "generate",
|
|
38
|
+
async run(opts, write) {
|
|
39
|
+
// Emit PRNG: the legacy build-anchor-absorption-shard.mjs seeded an LCG (lcg(opts.seed)).
|
|
40
|
+
const random = makeLcg(opts.seed);
|
|
41
|
+
const source = opts.sourceName ?? "synth-anchor-absorption";
|
|
42
|
+
const count = opts.count ?? 50000;
|
|
43
|
+
const realZips = loadRealUsZips(ANCHOR_LOOKUP);
|
|
44
|
+
console.error(`Loaded ${realZips.length} real US ZIPs from the anchor lookup (the leading-5-digit source).`);
|
|
45
|
+
let written = 0;
|
|
46
|
+
let quarantined = 0;
|
|
47
|
+
const byTemplate = {};
|
|
48
|
+
for (let i = 0; i < count; i++) {
|
|
49
|
+
const synth = synthesizeAnchorAbsorptionRow({ random, realZips });
|
|
50
|
+
const country = synth.locale.split("-")[1]; // "en-US" -> "US", "de-DE" -> "DE"
|
|
51
|
+
const canonical = {
|
|
52
|
+
raw: synth.raw,
|
|
53
|
+
components: synth.components,
|
|
54
|
+
country,
|
|
55
|
+
locale: synth.locale,
|
|
56
|
+
source,
|
|
57
|
+
source_id: stableSourceID(source, `${i}`),
|
|
58
|
+
};
|
|
59
|
+
const aligned = alignRow(canonical);
|
|
60
|
+
if (aligned.kind !== "labeled") {
|
|
61
|
+
quarantined++;
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
write(JSON.stringify({ ...aligned.row, synth_method: "anchor-absorption", synth_template: synth.template }) + "\n");
|
|
65
|
+
written++;
|
|
66
|
+
byTemplate[synth.template] = (byTemplate[synth.template] ?? 0) + 1;
|
|
67
|
+
}
|
|
68
|
+
console.error(`\nwrote ${written} rows (${quarantined} quarantined)`);
|
|
69
|
+
console.error(" by slice:", JSON.stringify(byTemplate));
|
|
70
|
+
return { emitted: written, skipped: quarantined };
|
|
71
|
+
},
|
|
72
|
+
};
|
|
73
|
+
//# sourceMappingURL=anchor-absorption.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"anchor-absorption.js","sourceRoot":"","sources":["../../../src/shard-recipes/anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AAEtC,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEpD,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAAE,6BAA6B,EAAE,MAAM,oCAAoC,CAAA;AAClF,OAAO,EAAE,OAAO,EAAoB,MAAM,eAAe,CAAA;AAEzD,uGAAuG;AACvG,wFAAwF;AACxF,MAAM,aAAa,GAAG,YAAY,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;AAExE,qGAAqG;AACrG,SAAS,cAAc,CAAC,IAAY;IACnC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAA;IAC3E,MAAM,IAAI,GAAa,EAAE,CAAA;IAEzB,KAAK,MAAM,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;QACzC,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,IAAI,IAAK,CAAC,CAAC,CAAC,CAAY,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC;YAC5G,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,CAAC;IACF,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,MAAM,CAAC,MAAM,sBAAsB,GAAgB;IAClD,IAAI,EAAE,mBAAmB;IACzB,WAAW,EAAE,iGAAiG;IAC9G,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,0FAA0F;QAC1F,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,yBAAyB,CAAA;QAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,CAAC,CAAA;QAC9C,OAAO,CAAC,KAAK,CAAC,UAAU,QAAQ,CAAC,MAAM,oEAAoE,CAAC,CAAA;QAE5G,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,WAAW,GAAG,CAAC,CAAA;QACnB,MAAM,UAAU,GAA2B,EAAE,CAAA;QAE7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,KAAK,GAAG,6BAA6B,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAA;YACjE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAA,CAAC,mCAAmC;YAC9E,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,UAAU,EAAE,KAAK,CAAC,UAAU;gBAC5B,OAAO;gBACP,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,MAAM;gBACN,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,EAAqD,CAAC;aAC5F,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,WAAW,EAAE,CAAA;gBACb,SAAQ;YACT,CAAC;YACD,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,EAAE,cAAc,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,GAAG,IAAI,CAC5G,CAAA;YACD,OAAO,EAAE,CAAA;YACT,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnE,CAAC;QACD,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,UAAU,WAAW,eAAe,CAAC,CAAA;QACrE,OAAO,CAAC,KAAK,CAAC,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC,CAAA;QAExD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,CAAA;IAClD,CAAC;CACD,CAAA"}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `boundary-stress` shard recipe (#375) — the boundary-instability augmentation. Self-generates
|
|
7
|
+
* `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
|
|
8
|
+
* DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
|
|
9
|
+
* lever for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
|
|
10
|
+
* scripts/build-boundary-stress-shard.mjs.
|
|
11
|
+
*
|
|
12
|
+
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
|
+
*/
|
|
14
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
15
|
+
export declare const boundaryStressRecipe: ShardRecipe;
|
|
16
|
+
//# sourceMappingURL=boundary-stress.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-stress.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH,OAAO,EAAkB,KAAK,WAAW,EAAiB,MAAM,eAAe,CAAA;AA8B/E,eAAO,MAAM,oBAAoB,EAAE,WAwClC,CAAA"}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `boundary-stress` shard recipe (#375) — the boundary-instability augmentation. Self-generates
|
|
7
|
+
* `--count` rows from {@link synthesizeBoundaryStressRow}'s weighted template mix (the v1.7.0,
|
|
8
|
+
* DeepSeek-tuned 2026-06-18 composition), aligns each to BIO, and emits a labeled JSONL. The
|
|
9
|
+
* lever for the taxonomy's #1 parser family (the boundary-wobble class). Ported from
|
|
10
|
+
* scripts/build-boundary-stress-shard.mjs.
|
|
11
|
+
*
|
|
12
|
+
* `synthesizeBoundaryStressRow` is NOT re-exported from the corpus index — imported directly here.
|
|
13
|
+
*/
|
|
14
|
+
import { alignRow } from "../align.js";
|
|
15
|
+
import { synthesizeBoundaryStressRow } from "../synthesize-boundary-stress.js";
|
|
16
|
+
import { makeMulberry32, shardSourceID } from "./scaffold.js";
|
|
17
|
+
// Revised composition (v1.7.0, DeepSeek-tuned 2026-06-18): `bare-locality` ~11% (recover the 84% locality
|
|
18
|
+
// drop on bare "City, STATE" rows WITHOUT becoming a locality-first majority), and
|
|
19
|
+
// house-number-before:after = 7:3 (FR's dominant order is number-BEFORE; 30% after breaks the order-bias
|
|
20
|
+
// shortcut without risking FR hn-before accuracy). The three original non-number shapes keep the bulk.
|
|
21
|
+
// Weights sum to 1.0. Key order is load-bearing — it drives the cumulative thresholds below.
|
|
22
|
+
const WEIGHTS = {
|
|
23
|
+
"street-eats-affix": 0.22,
|
|
24
|
+
"comma-less-city-state": 0.22,
|
|
25
|
+
"fr-prefix": 0.18,
|
|
26
|
+
"bare-locality": 0.11,
|
|
27
|
+
"house-number-before-street": 0.189,
|
|
28
|
+
"house-number-after-street": 0.081,
|
|
29
|
+
};
|
|
30
|
+
const CUM = (() => {
|
|
31
|
+
let acc = 0;
|
|
32
|
+
return Object.entries(WEIGHTS).map(([t, w]) => [t, (acc += w)]);
|
|
33
|
+
})();
|
|
34
|
+
function pickTemplate(r) {
|
|
35
|
+
const x = r();
|
|
36
|
+
for (const [t, c] of CUM)
|
|
37
|
+
if (x <= c)
|
|
38
|
+
return t;
|
|
39
|
+
return CUM[CUM.length - 1][0];
|
|
40
|
+
}
|
|
41
|
+
export const boundaryStressRecipe = {
|
|
42
|
+
name: "boundary-stress",
|
|
43
|
+
description: "Boundary-instability rows (#375): weighted template mix → synthesizeBoundaryStressRow → aligned BIO",
|
|
44
|
+
mode: "generate",
|
|
45
|
+
async run(opts, write) {
|
|
46
|
+
// Emit PRNG: the legacy build-boundary-stress-shard.mjs seeded mulberry32(opts.seed).
|
|
47
|
+
const random = makeMulberry32(opts.seed);
|
|
48
|
+
const count = opts.count ?? 20000;
|
|
49
|
+
let emitted = 0;
|
|
50
|
+
let skipped = 0;
|
|
51
|
+
for (let i = 0; i < count; i++) {
|
|
52
|
+
const row = synthesizeBoundaryStressRow(undefined, { random, forceTemplate: pickTemplate(random) });
|
|
53
|
+
const country = row.locale.split("-")[1] ?? "US";
|
|
54
|
+
const source_id = shardSourceID("synth-boundary-stress", { ...row.components, v: String(i) });
|
|
55
|
+
const canonical = {
|
|
56
|
+
raw: row.raw,
|
|
57
|
+
components: row.components,
|
|
58
|
+
country,
|
|
59
|
+
locale: row.locale,
|
|
60
|
+
source: "synth-boundary-stress",
|
|
61
|
+
source_id,
|
|
62
|
+
corpus_version: "0.6.0",
|
|
63
|
+
license: "Synthetic — boundary-stress; derived from public-domain locality/region tuples",
|
|
64
|
+
};
|
|
65
|
+
const aligned = alignRow(canonical);
|
|
66
|
+
if (aligned.kind !== "labeled") {
|
|
67
|
+
skipped++;
|
|
68
|
+
continue;
|
|
69
|
+
}
|
|
70
|
+
// Match the base corpus parquet schema: flat synth_method / synth_base_id, not a nested `synth`.
|
|
71
|
+
write(JSON.stringify({ ...aligned.row, synth_method: `boundary-stress:${row.template}`, synth_base_id: null }) + "\n");
|
|
72
|
+
emitted++;
|
|
73
|
+
}
|
|
74
|
+
return { emitted, skipped };
|
|
75
|
+
},
|
|
76
|
+
};
|
|
77
|
+
//# sourceMappingURL=boundary-stress.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"boundary-stress.js","sourceRoot":"","sources":["../../../src/shard-recipes/boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AACtC,OAAO,EAA+B,2BAA2B,EAAE,MAAM,kCAAkC,CAAA;AAC3G,OAAO,EAAE,cAAc,EAAoB,aAAa,EAAE,MAAM,eAAe,CAAA;AAE/E,0GAA0G;AAC1G,mFAAmF;AACnF,yGAAyG;AACzG,uGAAuG;AACvG,6FAA6F;AAC7F,MAAM,OAAO,GAA2C;IACvD,mBAAmB,EAAE,IAAI;IACzB,uBAAuB,EAAE,IAAI;IAC7B,WAAW,EAAE,IAAI;IACjB,eAAe,EAAE,IAAI;IACrB,4BAA4B,EAAE,KAAK;IACnC,2BAA2B,EAAE,KAAK;CAClC,CAAA;AACD,MAAM,GAAG,GAA4C,CAAC,GAAG,EAAE;IAC1D,IAAI,GAAG,GAAG,CAAC,CAAA;IAEX,OAAQ,MAAM,CAAC,OAAO,CAAC,OAAO,CAA6C,CAAC,GAAG,CAC9E,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAqC,CAC/D,CAAA;AACF,CAAC,CAAC,EAAE,CAAA;AACJ,SAAS,YAAY,CAAC,CAAe;IACpC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAA;IAEb,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG;QAAE,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,CAAC,CAAA;IAE9C,OAAO,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;AAC/B,CAAC;AAED,MAAM,CAAC,MAAM,oBAAoB,GAAgB;IAChD,IAAI,EAAE,iBAAiB;IACvB,WAAW,EAAE,qGAAqG;IAClH,IAAI,EAAE,UAAU;IAChB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,sFAAsF;QACtF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,KAAK,CAAA;QACjC,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,MAAM,GAAG,GAAG,2BAA2B,CAAC,SAAS,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,CAAA;YACnG,MAAM,OAAO,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;YAChD,MAAM,SAAS,GAAG,aAAa,CAAC,uBAAuB,EAAE,EAAE,GAAG,GAAG,CAAC,UAAU,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;YAC7F,MAAM,SAAS,GAAG;gBACjB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU,EAAE,GAAG,CAAC,UAAU;gBAC1B,OAAO;gBACP,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,MAAM,EAAE,uBAAuB;gBAC/B,SAAS;gBACT,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAA2C,CAAC,CAAA;YAErE,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;gBAChC,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,iGAAiG;YACjG,KAAK,CACJ,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,mBAAmB,GAAG,CAAC,QAAQ,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAC/G,CAAA;YACD,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|