@mailwoman/corpus 4.16.2 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +23 -26
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +25 -28
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +10 -10
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts +2 -2
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +2 -2
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +2 -2
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +9 -9
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +5 -5
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +3 -3
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +3 -3
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +11 -10
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts +4 -5
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +1 -1
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/index.d.ts +9 -10
- package/out/src/adapters/index.d.ts.map +1 -1
- package/out/src/adapters/index.js +13 -14
- package/out/src/adapters/index.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts +8 -10
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +8 -8
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +2 -2
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +6 -6
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +6 -6
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +6 -6
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +7 -7
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +9 -9
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +6 -6
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/adapter.d.ts.map +1 -1
- package/out/src/adapters/tiger/adapter.js +2 -2
- package/out/src/adapters/tiger/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts +2 -2
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +2 -2
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +15 -17
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +6 -6
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +5 -5
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +6 -6
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +7 -7
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +12 -12
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +3 -3
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +6 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +17 -19
- package/out/src/adapters/wof-admin-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-json/adapter.js +25 -28
- package/out/src/adapters/wof-admin-json/adapter.js.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +8 -8
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-postalcode-json/adapter.js +11 -11
- package/out/src/adapters/wof-postalcode-json/adapter.js.map +1 -1
- package/out/src/align.d.ts +14 -16
- package/out/src/align.d.ts.map +1 -1
- package/out/src/align.js +15 -16
- package/out/src/align.js.map +1 -1
- package/out/src/build.d.ts +13 -15
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +4 -5
- package/out/src/build.js.map +1 -1
- package/out/src/codex/us-fips-state.d.ts +6 -8
- package/out/src/codex/us-fips-state.d.ts.map +1 -1
- package/out/src/codex/us-fips-state.js +6 -8
- package/out/src/codex/us-fips-state.js.map +1 -1
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +1 -1
- package/out/src/golden.js.map +1 -1
- package/out/src/index.d.ts +1 -0
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/license.d.ts +5 -6
- package/out/src/license.d.ts.map +1 -1
- package/out/src/license.js +5 -6
- package/out/src/license.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +2 -2
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +16 -18
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +22 -23
- package/out/src/parquet.js.map +1 -1
- package/out/src/runner.d.ts +10 -10
- package/out/src/runner.d.ts.map +1 -1
- package/out/src/runner.js +8 -8
- package/out/src/runner.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts +16 -0
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -0
- package/out/src/shard-recipes/anchor-absorption.js +73 -0
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -0
- package/out/src/shard-recipes/boundary-stress.d.ts +16 -0
- package/out/src/shard-recipes/boundary-stress.d.ts.map +1 -0
- package/out/src/shard-recipes/boundary-stress.js +77 -0
- package/out/src/shard-recipes/boundary-stress.js.map +1 -0
- package/out/src/shard-recipes/country-balanced.d.ts +28 -0
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -0
- package/out/src/shard-recipes/country-balanced.js +370 -0
- package/out/src/shard-recipes/country-balanced.js.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts +34 -0
- package/out/src/shard-recipes/fr-admin-split.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-admin-split.js +175 -0
- package/out/src/shard-recipes/fr-admin-split.js.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts +24 -0
- package/out/src/shard-recipes/fr-bare-street.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-bare-street.js +76 -0
- package/out/src/shard-recipes/fr-bare-street.js.map +1 -0
- package/out/src/shard-recipes/fr-order.d.ts +34 -0
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -0
- package/out/src/shard-recipes/fr-order.js +238 -0
- package/out/src/shard-recipes/fr-order.js.map +1 -0
- package/out/src/shard-recipes/german.d.ts +19 -0
- package/out/src/shard-recipes/german.d.ts.map +1 -0
- package/out/src/shard-recipes/german.js +170 -0
- package/out/src/shard-recipes/german.js.map +1 -0
- package/out/src/shard-recipes/house-venue.d.ts +14 -0
- package/out/src/shard-recipes/house-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/house-venue.js +66 -0
- package/out/src/shard-recipes/house-venue.js.map +1 -0
- package/out/src/shard-recipes/index.d.ts +18 -0
- package/out/src/shard-recipes/index.d.ts.map +1 -0
- package/out/src/shard-recipes/index.js +58 -0
- package/out/src/shard-recipes/index.js.map +1 -0
- package/out/src/shard-recipes/intersection.d.ts +38 -0
- package/out/src/shard-recipes/intersection.d.ts.map +1 -0
- package/out/src/shard-recipes/intersection.js +473 -0
- package/out/src/shard-recipes/intersection.js.map +1 -0
- package/out/src/shard-recipes/locale.d.ts +19 -0
- package/out/src/shard-recipes/locale.d.ts.map +1 -0
- package/out/src/shard-recipes/locale.js +257 -0
- package/out/src/shard-recipes/locale.js.map +1 -0
- package/out/src/shard-recipes/no-street.d.ts +14 -0
- package/out/src/shard-recipes/no-street.d.ts.map +1 -0
- package/out/src/shard-recipes/no-street.js +66 -0
- package/out/src/shard-recipes/no-street.js.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts +34 -0
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box-cedex.js +686 -0
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -0
- package/out/src/shard-recipes/po-box.d.ts +13 -0
- package/out/src/shard-recipes/po-box.d.ts.map +1 -0
- package/out/src/shard-recipes/po-box.js +93 -0
- package/out/src/shard-recipes/po-box.js.map +1 -0
- package/out/src/shard-recipes/scaffold.d.ts +104 -0
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -0
- package/out/src/shard-recipes/scaffold.js +76 -0
- package/out/src/shard-recipes/scaffold.js.map +1 -0
- package/out/src/shard-recipes/street-affix.d.ts +31 -0
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -0
- package/out/src/shard-recipes/street-affix.js +408 -0
- package/out/src/shard-recipes/street-affix.js.map +1 -0
- package/out/src/shard-recipes/street-bare.d.ts +21 -0
- package/out/src/shard-recipes/street-bare.d.ts.map +1 -0
- package/out/src/shard-recipes/street-bare.js +82 -0
- package/out/src/shard-recipes/street-bare.js.map +1 -0
- package/out/src/shard-recipes/street.d.ts +11 -0
- package/out/src/shard-recipes/street.d.ts.map +1 -0
- package/out/src/shard-recipes/street.js +63 -0
- package/out/src/shard-recipes/street.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +25 -0
- package/out/src/shard-recipes/unit.d.ts.map +1 -0
- package/out/src/shard-recipes/unit.js +255 -0
- package/out/src/shard-recipes/unit.js.map +1 -0
- package/out/src/split.d.ts +23 -29
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +22 -29
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-anchor-absorption.d.ts +2 -3
- package/out/src/synthesize-anchor-absorption.d.ts.map +1 -1
- package/out/src/synthesize-anchor-absorption.js +2 -2
- package/out/src/synthesize-anchor-absorption.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +3 -4
- package/out/src/synthesize-boundary-stress.d.ts.map +1 -1
- package/out/src/synthesize-boundary-stress.js +3 -4
- package/out/src/synthesize-boundary-stress.js.map +1 -1
- package/out/src/synthesize-german.d.ts +15 -19
- package/out/src/synthesize-german.d.ts.map +1 -1
- package/out/src/synthesize-german.js +14 -17
- package/out/src/synthesize-german.js.map +1 -1
- package/out/src/synthesize-house-venue.d.ts +2 -3
- package/out/src/synthesize-house-venue.d.ts.map +1 -1
- package/out/src/synthesize-house-venue.js +2 -3
- package/out/src/synthesize-house-venue.js.map +1 -1
- package/out/src/synthesize-intersection.d.ts +2 -2
- package/out/src/synthesize-intersection.d.ts.map +1 -1
- package/out/src/synthesize-intersection.js +4 -4
- package/out/src/synthesize-intersection.js.map +1 -1
- package/out/src/synthesize-no-street.d.ts +5 -5
- package/out/src/synthesize-no-street.d.ts.map +1 -1
- package/out/src/synthesize-no-street.js +27 -23
- package/out/src/synthesize-no-street.js.map +1 -1
- package/out/src/synthesize-po-box.d.ts +15 -17
- package/out/src/synthesize-po-box.d.ts.map +1 -1
- package/out/src/synthesize-po-box.js +27 -27
- package/out/src/synthesize-po-box.js.map +1 -1
- package/out/src/synthesize-street.d.ts +8 -9
- package/out/src/synthesize-street.d.ts.map +1 -1
- package/out/src/synthesize-street.js +3 -3
- package/out/src/synthesize-street.js.map +1 -1
- package/out/src/synthesize.d.ts +57 -66
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +56 -65
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tokenize.d.ts +5 -6
- package/out/src/tokenize.d.ts.map +1 -1
- package/out/src/tokenize.js +5 -6
- package/out/src/tokenize.js.map +1 -1
- package/out/src/types.d.ts +49 -51
- package/out/src/types.d.ts.map +1 -1
- package/out/src/wof-json.d.ts +32 -34
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +23 -25
- package/out/src/wof-json.js.map +1 -1
- package/package.json +14 -14
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `unit` shard recipe — US secondary-unit coverage (#451, the v0-parity `unit` gap). Onto REAL US
|
|
7
|
+
* OpenAddresses skeletons (cached zips under `/tmp/oa-cache`) it INJECTS a USPS Pub-28 Appendix
|
|
8
|
+
* C2 secondary-unit designator (the `@mailwoman/codex/us` table), varying the surface form
|
|
9
|
+
* (canonical "Apartment" vs approved "Apt") AND the unit's POSITION (after-street / unit-first /
|
|
10
|
+
* bare / venue-prefixed) per row, so the model learns to RECOGNIZE the designator wherever it
|
|
11
|
+
* sits. The inline synthesis (the OA-CSV reader, the designator tables, `makeUnit`/`renderUnit`)
|
|
12
|
+
* is ported faithfully from scripts/build-unit-shard.mjs.
|
|
13
|
+
*
|
|
14
|
+
* `--golden`: a held-out eval over the VERMONT source only (the corpus `defaultHoldout`, never
|
|
15
|
+
* trained) with a different seed, emitting `{raw, components, country}` for per-locale-f1. Train
|
|
16
|
+
* uses every NON-Vermont US source. Designators are injected in both (OA carries none), so the
|
|
17
|
+
* eval measures designator recognition on held-out addresses.
|
|
18
|
+
*
|
|
19
|
+
* NOTE: this is a `generate`-mode recipe but it still reads REAL tuples off disk (`unzip` of the
|
|
20
|
+
* cached OA zips) — `--count` bounds the OUTPUT, not the input. The passed `random` (the
|
|
21
|
+
* framework LCG) is consumed in the exact call order the legacy script used.
|
|
22
|
+
*/
|
|
23
|
+
import { type ShardRecipe } from "./scaffold.js";
|
|
24
|
+
export declare const unitRecipe: ShardRecipe;
|
|
25
|
+
//# sourceMappingURL=unit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unit.d.ts","sourceRoot":"","sources":["../../../src/shard-recipes/unit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAUH,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,eAAe,CAAA;AA6MhE,eAAO,MAAM,UAAU,EAAE,WAqExB,CAAA"}
|
|
@@ -0,0 +1,255 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* `unit` shard recipe — US secondary-unit coverage (#451, the v0-parity `unit` gap). Onto REAL US
|
|
7
|
+
* OpenAddresses skeletons (cached zips under `/tmp/oa-cache`) it INJECTS a USPS Pub-28 Appendix
|
|
8
|
+
* C2 secondary-unit designator (the `@mailwoman/codex/us` table), varying the surface form
|
|
9
|
+
* (canonical "Apartment" vs approved "Apt") AND the unit's POSITION (after-street / unit-first /
|
|
10
|
+
* bare / venue-prefixed) per row, so the model learns to RECOGNIZE the designator wherever it
|
|
11
|
+
* sits. The inline synthesis (the OA-CSV reader, the designator tables, `makeUnit`/`renderUnit`)
|
|
12
|
+
* is ported faithfully from scripts/build-unit-shard.mjs.
|
|
13
|
+
*
|
|
14
|
+
* `--golden`: a held-out eval over the VERMONT source only (the corpus `defaultHoldout`, never
|
|
15
|
+
* trained) with a different seed, emitting `{raw, components, country}` for per-locale-f1. Train
|
|
16
|
+
* uses every NON-Vermont US source. Designators are injected in both (OA carries none), so the
|
|
17
|
+
* eval measures designator recognition on held-out addresses.
|
|
18
|
+
*
|
|
19
|
+
* NOTE: this is a `generate`-mode recipe but it still reads REAL tuples off disk (`unzip` of the
|
|
20
|
+
* cached OA zips) — `--count` bounds the OUTPUT, not the input. The passed `random` (the
|
|
21
|
+
* framework LCG) is consumed in the exact call order the legacy script used.
|
|
22
|
+
*/
|
|
23
|
+
import { spawnSync } from "node:child_process";
|
|
24
|
+
import { US_UNIT_DESIGNATOR_PREFERRED_ABBR } from "@mailwoman/codex/us";
|
|
25
|
+
import { stableSourceID } from "../adapter.js";
|
|
26
|
+
import { alignRow } from "../align.js";
|
|
27
|
+
import { makeMulberry32 } from "./scaffold.js";
|
|
28
|
+
// OA REGION is empty for US per-state extracts — the region is implied by the file. Train sources are
|
|
29
|
+
// every NON-Vermont state cached; eval is Vermont only (the corpus holdout).
|
|
30
|
+
const TRAIN_SOURCES = [
|
|
31
|
+
{ zip: "/tmp/oa-cache/us__ca__berkeley.zip", csv: "us/ca/berkeley.csv", region: "CA" },
|
|
32
|
+
{ zip: "/tmp/oa-cache/us__ca__marin.zip", csv: "us/ca/marin.csv", region: "CA" },
|
|
33
|
+
{ zip: "/tmp/oa-cache/us__dc__statewide.zip", csv: "us/dc/statewide.csv", region: "DC" },
|
|
34
|
+
{ zip: "/tmp/oa-cache/us__ia__statewide.zip", csv: "us/ia/statewide.csv", region: "IA" },
|
|
35
|
+
{ zip: "/tmp/oa-cache/us__il__cook.zip", csv: "us/il/cook.csv", region: "IL" },
|
|
36
|
+
{ zip: "/tmp/oa-cache/us__mt__statewide.zip", csv: "us/mt/statewide.csv", region: "MT" },
|
|
37
|
+
{ zip: "/tmp/oa-cache/us__sd__statewide.zip", csv: "us/sd/statewide.csv", region: "SD" },
|
|
38
|
+
];
|
|
39
|
+
const EVAL_SOURCE = { zip: "/tmp/oa-cache/us__vt__statewide.zip", csv: "us/vt/statewide.csv", region: "VT" };
|
|
40
|
+
// USPS Pub-28 C2 designators that take a secondary identifier ("Apt 4B"). Weighted toward the common
|
|
41
|
+
// ones the v0-parity arena failed on (Apt/Ste/Unit/Fl/Rm). Standalone designators (Basement, Lobby,
|
|
42
|
+
// Penthouse) are emitted occasionally with no id.
|
|
43
|
+
const ID_DESIGNATORS = [
|
|
44
|
+
"APARTMENT",
|
|
45
|
+
"SUITE",
|
|
46
|
+
"UNIT",
|
|
47
|
+
"FLOOR",
|
|
48
|
+
"ROOM",
|
|
49
|
+
"BUILDING",
|
|
50
|
+
"DEPARTMENT",
|
|
51
|
+
"SPACE",
|
|
52
|
+
"LOT",
|
|
53
|
+
];
|
|
54
|
+
const STANDALONE_DESIGNATORS = [
|
|
55
|
+
"BASEMENT",
|
|
56
|
+
"LOBBY",
|
|
57
|
+
"PENTHOUSE",
|
|
58
|
+
"FRONT",
|
|
59
|
+
"REAR",
|
|
60
|
+
"UPPER",
|
|
61
|
+
"LOWER",
|
|
62
|
+
];
|
|
63
|
+
const ID_WEIGHT = 0.85; // 85% id-bearing designators, 15% standalone
|
|
64
|
+
const SYNTH_IDS = ["4B", "200", "12", "3", "A", "101", "5", "2A", "310", "B", "7", "1500", "404"];
|
|
65
|
+
/** Minimal RFC-4180-ish splitter (handles quoted fields). */
|
|
66
|
+
function splitCSV(line) {
|
|
67
|
+
const out = [];
|
|
68
|
+
let cur = "";
|
|
69
|
+
let inQ = false;
|
|
70
|
+
for (let i = 0; i < line.length; i++) {
|
|
71
|
+
const c = line[i];
|
|
72
|
+
if (inQ) {
|
|
73
|
+
if (c === '"') {
|
|
74
|
+
if (line[i + 1] === '"') {
|
|
75
|
+
cur += '"';
|
|
76
|
+
i++;
|
|
77
|
+
}
|
|
78
|
+
else
|
|
79
|
+
inQ = false;
|
|
80
|
+
}
|
|
81
|
+
else
|
|
82
|
+
cur += c;
|
|
83
|
+
}
|
|
84
|
+
else if (c === '"')
|
|
85
|
+
inQ = true;
|
|
86
|
+
else if (c === ",") {
|
|
87
|
+
out.push(cur);
|
|
88
|
+
cur = "";
|
|
89
|
+
}
|
|
90
|
+
else
|
|
91
|
+
cur += c;
|
|
92
|
+
}
|
|
93
|
+
out.push(cur);
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Stream real US tuples (number/street/city/postcode + the bare OA unit id) out of a cached OA zip. */
|
|
97
|
+
function readTuples(source) {
|
|
98
|
+
const r = spawnSync("unzip", ["-p", source.zip, source.csv], { maxBuffer: 1024 * 1024 * 1024, encoding: "buffer" });
|
|
99
|
+
if (r.status !== 0) {
|
|
100
|
+
console.error(` WARN: unzip failed for ${source.zip} (status ${r.status})`);
|
|
101
|
+
return [];
|
|
102
|
+
}
|
|
103
|
+
const lines = r.stdout.toString("utf8").split(/\r?\n/);
|
|
104
|
+
if (lines.length < 2)
|
|
105
|
+
return [];
|
|
106
|
+
const header = splitCSV(lines[0]).map((h) => h.trim().toLowerCase());
|
|
107
|
+
const idx = (name) => header.indexOf(name);
|
|
108
|
+
const iNum = idx("number"), iStreet = idx("street"), iUnit = idx("unit"), iCity = idx("city"), iPost = idx("postcode");
|
|
109
|
+
const get = (cells, i) => (i >= 0 && i < cells.length ? (cells[i] ?? "").trim() : "");
|
|
110
|
+
const tuples = [];
|
|
111
|
+
const seen = new Set();
|
|
112
|
+
for (let li = 1; li < lines.length; li++) {
|
|
113
|
+
if (!lines[li])
|
|
114
|
+
continue;
|
|
115
|
+
const cells = splitCSV(lines[li]);
|
|
116
|
+
const street = get(cells, iStreet);
|
|
117
|
+
const locality = get(cells, iCity);
|
|
118
|
+
const house_number = get(cells, iNum);
|
|
119
|
+
if (!street || !locality || !house_number)
|
|
120
|
+
continue;
|
|
121
|
+
const key = `${house_number}|${street}|${locality}`.toLowerCase();
|
|
122
|
+
if (seen.has(key))
|
|
123
|
+
continue;
|
|
124
|
+
seen.add(key);
|
|
125
|
+
tuples.push({
|
|
126
|
+
house_number,
|
|
127
|
+
street,
|
|
128
|
+
locality,
|
|
129
|
+
region: source.region,
|
|
130
|
+
postcode: get(cells, iPost),
|
|
131
|
+
oaUnit: get(cells, iUnit),
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
return tuples;
|
|
135
|
+
}
|
|
136
|
+
/** Title-case a canonical/abbrev designator ("APARTMENT" → "Apartment", "APT" → "Apt"). */
|
|
137
|
+
const title = (s) => s.charAt(0).toUpperCase() + s.slice(1).toLowerCase();
|
|
138
|
+
/** Build an injected unit string ("Apt 4B"), varying canonical vs approved-abbrev form per row. */
|
|
139
|
+
function makeUnit(random, oaUnit) {
|
|
140
|
+
const standalone = random() >= ID_WEIGHT;
|
|
141
|
+
const pool = standalone ? STANDALONE_DESIGNATORS : ID_DESIGNATORS;
|
|
142
|
+
const canonical = pool[Math.floor(random() * pool.length)];
|
|
143
|
+
// Vary the surface form 50/50 (this is the #454 expand/abbreviate variety, baked into the shard).
|
|
144
|
+
const designator = random() < 0.5 ? title(canonical) : title(US_UNIT_DESIGNATOR_PREFERRED_ABBR[canonical]);
|
|
145
|
+
if (standalone)
|
|
146
|
+
return designator;
|
|
147
|
+
const id = oaUnit && oaUnit.length <= 6 ? oaUnit : SYNTH_IDS[Math.floor(random() * SYNTH_IDS.length)];
|
|
148
|
+
return `${designator} ${id}`;
|
|
149
|
+
}
|
|
150
|
+
/** Synthetic recipient/venue prefixes — the "JOHN DOE, ACME INC, ..." arena pattern. */
|
|
151
|
+
const VENUES = [
|
|
152
|
+
"John Doe",
|
|
153
|
+
"Jane Smith",
|
|
154
|
+
"Acme Inc",
|
|
155
|
+
"Wayne Enterprises",
|
|
156
|
+
"Stark Industries",
|
|
157
|
+
"Globex Corp",
|
|
158
|
+
"Maria Garcia",
|
|
159
|
+
"Robert Chen",
|
|
160
|
+
"Oak Street Dental",
|
|
161
|
+
"Riverside Clinic",
|
|
162
|
+
];
|
|
163
|
+
/** Address tail: "City, ST 12345" (or no postcode). */
|
|
164
|
+
const tail = (loc, reg, pc) => (pc ? `${loc}, ${reg} ${pc}` : `${loc}, ${reg}`);
|
|
165
|
+
/**
|
|
166
|
+
* Render a unit row in a RANDOM layout — units spread across positions, the city/state tail dropped on bare rows, a
|
|
167
|
+
* recipient/venue prefixed on the venue format — so the model learns to RECOGNIZE the designator wherever it sits.
|
|
168
|
+
* Returns {fmt, raw, components}.
|
|
169
|
+
*/
|
|
170
|
+
function renderUnit(random, base, unit) {
|
|
171
|
+
const hn = base.house_number, street = base.street, loc = base.locality, reg = base.region, pc = base.postcode;
|
|
172
|
+
const road = `${hn} ${street}`;
|
|
173
|
+
const full = {
|
|
174
|
+
house_number: hn,
|
|
175
|
+
street,
|
|
176
|
+
unit,
|
|
177
|
+
locality: loc,
|
|
178
|
+
region: reg,
|
|
179
|
+
...(pc ? { postcode: pc } : {}),
|
|
180
|
+
};
|
|
181
|
+
const r = random();
|
|
182
|
+
if (r < 0.34)
|
|
183
|
+
return { fmt: "full-after", raw: `${road} ${unit}, ${tail(loc, reg, pc)}`, components: full };
|
|
184
|
+
if (r < 0.52)
|
|
185
|
+
return { fmt: "full-first", raw: `${unit}, ${road}, ${tail(loc, reg, pc)}`, components: full };
|
|
186
|
+
if (r < 0.68)
|
|
187
|
+
return { fmt: "bare-after", raw: `${road} ${unit}`, components: { house_number: hn, street, unit } };
|
|
188
|
+
if (r < 0.84)
|
|
189
|
+
return { fmt: "bare-first", raw: `${unit} ${road}`, components: { house_number: hn, street, unit } };
|
|
190
|
+
const v = VENUES[Math.floor(random() * VENUES.length)];
|
|
191
|
+
return { fmt: "venue", raw: `${v}, ${road} ${unit}, ${tail(loc, reg, pc)}`, components: { venue: v, ...full } };
|
|
192
|
+
}
|
|
193
|
+
export const unitRecipe = {
|
|
194
|
+
name: "unit",
|
|
195
|
+
description: "US secondary-unit rows (#451): real OA skeletons + injected USPS Pub-28 C2 unit designators",
|
|
196
|
+
mode: "generate",
|
|
197
|
+
options: [{ flag: "--golden", description: "Emit the held-out VT eval slice ({raw, components, country})" }],
|
|
198
|
+
async run(opts, write) {
|
|
199
|
+
if (opts.count == null)
|
|
200
|
+
throw new Error("unit recipe requires --count <N>");
|
|
201
|
+
const count = opts.count;
|
|
202
|
+
// Legacy build-unit-shard.mjs seeded mulberry32 with the raw seed: `const random = mulberry32(opts.seed)`.
|
|
203
|
+
const random = makeMulberry32(opts.seed);
|
|
204
|
+
const source = opts.sourceName ?? "synth-unit";
|
|
205
|
+
const sources = opts.golden ? [EVAL_SOURCE] : TRAIN_SOURCES;
|
|
206
|
+
const pool = [];
|
|
207
|
+
for (const s of sources) {
|
|
208
|
+
const t = readTuples(s);
|
|
209
|
+
console.error(` ${s.csv}: ${t.length} unique tuples`);
|
|
210
|
+
for (const x of t)
|
|
211
|
+
pool.push(x);
|
|
212
|
+
}
|
|
213
|
+
if (pool.length === 0) {
|
|
214
|
+
throw new Error("No US tuples found — are the cached OA zips present in /tmp/oa-cache?");
|
|
215
|
+
}
|
|
216
|
+
let emitted = 0;
|
|
217
|
+
let skipped = 0;
|
|
218
|
+
let guard = 0;
|
|
219
|
+
const N = pool.length;
|
|
220
|
+
while (emitted < count && guard++ < count * 6) {
|
|
221
|
+
const base = pool[Math.floor(random() * N)];
|
|
222
|
+
const unit = makeUnit(random, base.oaUnit);
|
|
223
|
+
const { raw, components } = renderUnit(random, base, unit);
|
|
224
|
+
// The unit must survive verbatim in raw, else alignment can't label it.
|
|
225
|
+
if (!raw.includes(unit)) {
|
|
226
|
+
skipped++;
|
|
227
|
+
continue;
|
|
228
|
+
}
|
|
229
|
+
if (opts.golden) {
|
|
230
|
+
write(JSON.stringify({ raw, components, country: "US" }) + "\n");
|
|
231
|
+
emitted++;
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
const canonical = {
|
|
235
|
+
raw,
|
|
236
|
+
components,
|
|
237
|
+
country: "US",
|
|
238
|
+
locale: "en-US",
|
|
239
|
+
source,
|
|
240
|
+
source_id: stableSourceID(source, components),
|
|
241
|
+
corpus_version: "0.4.0",
|
|
242
|
+
license: "OpenAddresses US (non-VT) skeletons + injected USPS Pub-28 C2 unit designators",
|
|
243
|
+
};
|
|
244
|
+
const aligned = alignRow(canonical);
|
|
245
|
+
if (aligned.kind !== "labeled" || !aligned.row) {
|
|
246
|
+
skipped++;
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
write(JSON.stringify({ ...aligned.row, synth_method: "unit", synth_base_id: null }) + "\n");
|
|
250
|
+
emitted++;
|
|
251
|
+
}
|
|
252
|
+
return { emitted, skipped };
|
|
253
|
+
},
|
|
254
|
+
};
|
|
255
|
+
//# sourceMappingURL=unit.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unit.js","sourceRoot":"","sources":["../../../src/shard-recipes/unit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAE9C,OAAO,EAAE,iCAAiC,EAAyB,MAAM,qBAAqB,CAAA;AAG9F,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAA;AAC9C,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEtC,OAAO,EAAE,cAAc,EAAoB,MAAM,eAAe,CAAA;AAShE,sGAAsG;AACtG,6EAA6E;AAC7E,MAAM,aAAa,GAA0B;IAC5C,EAAE,GAAG,EAAE,oCAAoC,EAAE,GAAG,EAAE,oBAAoB,EAAE,MAAM,EAAE,IAAI,EAAE;IACtF,EAAE,GAAG,EAAE,iCAAiC,EAAE,GAAG,EAAE,iBAAiB,EAAE,MAAM,EAAE,IAAI,EAAE;IAChF,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACxF,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACxF,EAAE,GAAG,EAAE,gCAAgC,EAAE,GAAG,EAAE,gBAAgB,EAAE,MAAM,EAAE,IAAI,EAAE;IAC9E,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;IACxF,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE;CACxF,CAAA;AACD,MAAM,WAAW,GAAe,EAAE,GAAG,EAAE,qCAAqC,EAAE,GAAG,EAAE,qBAAqB,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;AAExH,qGAAqG;AACrG,oGAAoG;AACpG,kDAAkD;AAClD,MAAM,cAAc,GAAgC;IACnD,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,UAAU;IACV,YAAY;IACZ,OAAO;IACP,KAAK;CACL,CAAA;AACD,MAAM,sBAAsB,GAAgC;IAC3D,UAAU;IACV,OAAO;IACP,WAAW;IACX,OAAO;IACP,MAAM;IACN,OAAO;IACP,OAAO;CACP,CAAA;AACD,MAAM,SAAS,GAAG,IAAI,CAAA,CAAC,6CAA6C;AACpE,MAAM,SAAS,GAAsB,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,CAAC,CAAA;AAYpH,6DAA6D;AAC7D,SAAS,QAAQ,CAAC,IAAY;IAC7B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,IAAI,GAAG,GAAG,EAAE,CAAA;IACZ,IAAI,GAAG,GAAG,KAAK,CAAA;IAEf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QAEjB,IAAI,GAAG,EAAE,CAAC;YACT,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBACf,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC;oBACzB,GAAG,IAAI,GAAG,CAAA;oBACV,CAAC,EAAE,CAAA;gBACJ,CAAC;;oBAAM,GAAG,GAAG,KAAK,CAAA;YACnB,CAAC;;gBAAM,GAAG,IAAI,CAAC,CAAA;QAChB,CAAC;aAAM,IAAI,CAAC,KAAK,GAAG;YAAE,GAAG,GAAG,IAAI,CAAA;aAC3B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YACb,GAAG,GAAG,EAAE,CAAA;QACT,CAAC;;YAAM,GAAG,IAAI,CAAC,CAAA;IAChB,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;IAEb,OAAO,GAAG,CAAA;AACX,CAAC;AAED,wGAAwG;AACxG,SAAS,UAAU,CAAC,MAAkB;IACrC,MAAM,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEnH,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,MAAM,CAAC,GAAG,YAAY,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;QAE5E,OAAO,EAAE,CAAA;IACV,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAEtD,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAA;IACrE,MAAM,GAAG,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC1D,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAC,EACzB,OAAO,GAAG,GAAG,CAAC,QAAQ,CAAC,EACvB,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EACnB,KAAK,GAAG,GAAG,CAAC,MAAM,CAAC,EACnB,KAAK,GAAG,GAAG,CAAC,UAAU,CAAC,CAAA;IACxB,MAAM,GAAG,GAAG,CAAC,KAAe,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAA;IAC/G,MAAM,MAAM,GAAgB,EAAE,CAAA;IAC9B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAA;IAE9B,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAAE,SAAQ;QACxB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAE,CAAC,CAAA;QAClC,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;QAClC,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,CAAA;QAClC,MAAM,YAAY,GAAG,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAA;QAErC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,IAAI,CAAC,YAAY;YAAE,SAAQ;QACnD,MAAM,GAAG,GAAG,GAAG,YAAY,IAAI,MAAM,IAAI,QAAQ,EAAE,CAAC,WAAW,EAAE,CAAA;QAEjE,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAC3B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,IAAI,CAAC;YACX,YAAY;YACZ,MAAM;YACN,QAAQ;YACR,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;YAC3B,MAAM,EAAE,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC;SACzB,CAAC,CAAA;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED,2FAA2F;AAC3F,MAAM,KAAK,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,EAAE,CAAA;AAEzF,mGAAmG;AACnG,SAAS,QAAQ,CAAC,MAAoB,EAAE,MAAc;IACrD,MAAM,UAAU,GAAG,MAAM,EAAE,IAAI,SAAS,CAAA;IACxC,MAAM,IAAI,GAAG,UAAU,CAAC,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,cAAc,CAAA;IACjE,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,CAAE,CAAA;IAC3D,kGAAkG;IAClG,MAAM,UAAU,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,iCAAiC,CAAC,SAAS,CAAC,CAAC,CAAA;IAE1G,IAAI,UAAU;QAAE,OAAO,UAAU,CAAA;IACjC,MAAM,EAAE,GAAG,MAAM,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC,CAAE,CAAA;IAEtG,OAAO,GAAG,UAAU,IAAI,EAAE,EAAE,CAAA;AAC7B,CAAC;AAED,wFAAwF;AACxF,MAAM,MAAM,GAAsB;IACjC,UAAU;IACV,YAAY;IACZ,UAAU;IACV,mBAAmB;IACnB,kBAAkB;IAClB,aAAa;IACb,cAAc;IACd,aAAa;IACb,mBAAmB;IACnB,kBAAkB;CAClB,CAAA;AAED,uDAAuD;AACvD,MAAM,IAAI,GAAG,CAAC,GAAW,EAAE,GAAW,EAAE,EAAU,EAAU,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,GAAG,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,GAAG,EAAE,CAAC,CAAA;AAE/G;;;;GAIG;AACH,SAAS,UAAU,CAClB,MAAoB,EACpB,IAAe,EACf,IAAY;IAEZ,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,EAC3B,MAAM,GAAG,IAAI,CAAC,MAAM,EACpB,GAAG,GAAG,IAAI,CAAC,QAAQ,EACnB,GAAG,GAAG,IAAI,CAAC,MAAM,EACjB,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAA;IACnB,MAAM,IAAI,GAAG,GAAG,EAAE,IAAI,MAAM,EAAE,CAAA;IAC9B,MAAM,IAAI,GAA0C;QACnD,YAAY,EAAE,EAAE;QAChB,MAAM;QACN,IAAI;QACJ,QAAQ,EAAE,GAAG;QACb,MAAM,EAAE,GAAG;QACX,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/B,CAAA;IACD,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;IAElB,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IAE3G,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,IAAI,KAAK,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,IAAI,EAAE,CAAA;IAE5G,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAA;IAElH,IAAI,CAAC,GAAG,IAAI;QAAE,OAAO,EAAE,GAAG,EAAE,YAAY,EAAE,GAAG,EAAE,GAAG,IAAI,IAAI,IAAI,EAAE,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAA;IAClH,MAAM,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAE,CAAA;IAEvD,OAAO,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,IAAI,EAAE,EAAE,CAAA;AAChH,CAAC;AAED,MAAM,CAAC,MAAM,UAAU,GAAgB;IACtC,IAAI,EAAE,MAAM;IACZ,WAAW,EAAE,6FAA6F;IAC1G,IAAI,EAAE,UAAU;IAChB,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,WAAW,EAAE,8DAA8D,EAAE,CAAC;IAC5G,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK;QACpB,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QAC3E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QACxB,2GAA2G;QAC3G,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,IAAI,YAAY,CAAA;QAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,aAAa,CAAA;QAE3D,MAAM,IAAI,GAAgB,EAAE,CAAA;QAE5B,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC,CAAA;YACvB,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,GAAG,KAAK,CAAC,CAAC,MAAM,gBAAgB,CAAC,CAAA;YAEtD,KAAK,MAAM,CAAC,IAAI,CAAC;gBAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChC,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,IAAI,KAAK,CAAC,uEAAuE,CAAC,CAAA;QACzF,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,OAAO,GAAG,CAAC,CAAA;QACf,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAA;QAErB,OAAO,OAAO,GAAG,KAAK,IAAI,KAAK,EAAE,GAAG,KAAK,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAE,CAAA;YAC5C,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;YAC1C,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,GAAG,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;YAE1D,wEAAwE;YACxE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;gBACzB,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YAED,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;gBAChE,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,MAAM,SAAS,GAAiB;gBAC/B,GAAG;gBACH,UAAU;gBACV,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,OAAO;gBACf,MAAM;gBACN,SAAS,EAAE,cAAc,CAAC,MAAM,EAAE,UAAU,CAAC;gBAC7C,cAAc,EAAE,OAAO;gBACvB,OAAO,EAAE,gFAAgF;aACzF,CAAA;YACD,MAAM,OAAO,GAAG,QAAQ,CAAC,SAAS,CAAC,CAAA;YAEnC,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;gBAChD,OAAO,EAAE,CAAA;gBACT,SAAQ;YACT,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAA;YAC3F,OAAO,EAAE,CAAA;QACV,CAAC;QAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IAC5B,CAAC;CACD,CAAA"}
|
package/out/src/split.d.ts
CHANGED
|
@@ -27,9 +27,9 @@ import type { CanonicalRow, LabeledRow } from "./types.js";
|
|
|
27
27
|
export type SplitName = "train" | "val" | "test";
|
|
28
28
|
export interface SplitOptions {
|
|
29
29
|
/**
|
|
30
|
-
* Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the
|
|
31
|
-
*
|
|
32
|
-
*
|
|
30
|
+
* Region-name → holdout policy, keyed by ISO 3166-1 alpha-2 country. The values are the region-component strings the
|
|
31
|
+
* splitter looks for in `row.components.region`. Override to change the holdout for an experiment; defaults to
|
|
32
|
+
* `defaultHoldouts()`.
|
|
33
33
|
*/
|
|
34
34
|
holdouts?: Record<string, readonly string[]>;
|
|
35
35
|
}
|
|
@@ -55,38 +55,34 @@ export interface SplitManifest {
|
|
|
55
55
|
*
|
|
56
56
|
* - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
|
|
57
57
|
* - FR: Corse, Lozère, Creuse (small departments / regions).
|
|
58
|
-
* - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training
|
|
59
|
-
*
|
|
60
|
-
*
|
|
61
|
-
*
|
|
62
|
-
* SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a
|
|
63
|
-
* holdout).
|
|
58
|
+
* - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
|
|
59
|
+
* the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
|
|
60
|
+
* shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
|
|
61
|
+
* their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
|
|
64
62
|
*/
|
|
65
63
|
export declare function defaultHoldouts(): Record<string, readonly string[]>;
|
|
66
64
|
type SplitInputRow = Pick<CanonicalRow, "source_id" | "country" | "corpus_version" | "components">;
|
|
67
65
|
/**
|
|
68
|
-
* Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
* regardless of caller.
|
|
66
|
+
* Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
|
|
67
|
+
* (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
|
|
68
|
+
* array-based path so the decision is stable regardless of caller.
|
|
72
69
|
*/
|
|
73
70
|
export declare function splitForRow(row: Pick<SplitInputRow, "source_id" | "country" | "components">, holdouts?: Record<string, readonly string[]>): SplitName;
|
|
74
71
|
/**
|
|
75
|
-
* Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are
|
|
76
|
-
*
|
|
72
|
+
* Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
|
|
73
|
+
* `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
|
|
77
74
|
*
|
|
78
|
-
* Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via
|
|
79
|
-
*
|
|
80
|
-
*
|
|
75
|
+
* Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
|
|
76
|
+
* streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
|
|
77
|
+
* split membership in heap.
|
|
81
78
|
*/
|
|
82
79
|
export declare function splitRows(rows: Iterable<SplitInputRow>, opts?: SplitOptions): SplitManifest;
|
|
83
80
|
/** Lightweight deterministic 0..(n-1) bucket from a string id. */
|
|
84
81
|
export declare function hashBucket(id: string, n: number): number;
|
|
85
82
|
/**
|
|
86
|
-
* Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
* corpus version.
|
|
83
|
+
* Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
|
|
84
|
+
* id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
|
|
85
|
+
* including holdouts + counts + corpus version.
|
|
90
86
|
*
|
|
91
87
|
* Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
|
|
92
88
|
*/
|
|
@@ -94,14 +90,12 @@ export declare function writeSplitManifests(manifest: SplitManifest, outputDir:
|
|
|
94
90
|
/** Type re-export for callers that want to ingest LabeledRow specifically. */
|
|
95
91
|
export type SplitInputLabeledRow = Pick<LabeledRow, "source_id" | "country" | "corpus_version" | "components">;
|
|
96
92
|
/**
|
|
97
|
-
* Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests +
|
|
98
|
-
*
|
|
99
|
-
*
|
|
100
|
-
* files that exceed in-memory thresholds.
|
|
93
|
+
* Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
|
|
94
|
+
* streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
|
|
95
|
+
* handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
|
|
101
96
|
*
|
|
102
|
-
* Used by `buildCorpus` after the align loop has already partitioned labeled rows into
|
|
103
|
-
*
|
|
104
|
-
* passed in (zero re-scan).
|
|
97
|
+
* Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
|
|
98
|
+
* via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
|
|
105
99
|
*/
|
|
106
100
|
export declare function writeSplitManifestsFromLabeledFiles(opts: {
|
|
107
101
|
labeledPaths: Record<SplitName, string>;
|
package/out/src/split.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;
|
|
1
|
+
{"version":3,"file":"split.d.ts","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AASH,OAAO,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,YAAY,CAAA;AAE1D,MAAM,MAAM,SAAS,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAA;AAEhD,MAAM,WAAW,YAAY;IAC5B;;;;OAIG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C;AAED,kDAAkD;AAClD,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAA;IACf,GAAG,EAAE,MAAM,EAAE,CAAA;IACb,IAAI,EAAE,MAAM,EAAE,CAAA;IACd,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;IAC3C,yEAAyE;IACzE,cAAc,EAAE,MAAM,CAAA;IACtB,sCAAsC;IACtC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CACnE;AAED;;;;;;;;;GASG;AACH,wBAAgB,eAAe,IAAI,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAMnE;AAED,KAAK,aAAa,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAElG;;;;GAIG;AACH,wBAAgB,WAAW,CAC1B,GAAG,EAAE,IAAI,CAAC,aAAa,EAAE,WAAW,GAAG,SAAS,GAAG,YAAY,CAAC,EAChE,QAAQ,GAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAqB,GAC7D,SAAS,CASX;AAED;;;;;;;GAOG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,aAAa,CAAC,EAAE,IAAI,GAAE,YAAiB,GAAG,aAAa,CA0B/F;AAED,kEAAkE;AAClE,wBAAgB,UAAU,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAMxD;AAED;;;;;;GAMG;AACH,wBAAsB,mBAAmB,CAAC,QAAQ,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAanG;AAED,8EAA8E;AAC9E,MAAM,MAAM,oBAAoB,GAAG,IAAI,CAAC,UAAU,EAAE,WAAW,GAAG,SAAS,GAAG,gBAAgB,GAAG,YAAY,CAAC,CAAA;AAE9G;;;;;;;GAOG;AACH,wBAAsB,mCAAmC,CAAC,IAAI,EAAE;IAC/D,YAAY,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACvC,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC,CAAA;CAC5C,GAAG,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC,CAmBnC"}
|
package/out/src/split.js
CHANGED
|
@@ -34,12 +34,10 @@ import { createInterface } from "node:readline";
|
|
|
34
34
|
*
|
|
35
35
|
* - US: Vermont, Wyoming, North Dakota (low density, easy to identify in WOF/admin sources).
|
|
36
36
|
* - FR: Corse, Lozère, Creuse (small departments / regions).
|
|
37
|
-
* - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
41
|
-
* SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a
|
|
42
|
-
* holdout).
|
|
37
|
+
* - DE (added 2026-06-11, night-11): Saarland + Mecklenburg-Vorpommern — small Länder so the training cost is low while
|
|
38
|
+
* the slice clears the honest-eval 1000-row trust floor. DE has had NO trustable honest-eval slice since the harness
|
|
39
|
+
* shipped (flagged 2026-06-08); this takes effect at the NEXT base corpus rebuild — existing versioned corpora keep
|
|
40
|
+
* their committed SPLIT_MANIFESTs (a holdout added after a corpus is built is leakage-laundering, not a holdout).
|
|
43
41
|
*/
|
|
44
42
|
export function defaultHoldouts() {
|
|
45
43
|
return {
|
|
@@ -49,10 +47,9 @@ export function defaultHoldouts() {
|
|
|
49
47
|
};
|
|
50
48
|
}
|
|
51
49
|
/**
|
|
52
|
-
* Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* regardless of caller.
|
|
50
|
+
* Pure per-row split decision. Used by both the in-memory `splitRows` and by the streaming `buildCorpus` align loop
|
|
51
|
+
* (`build.ts`) to decide each row's split without retaining the row in heap. Identical hash bucketing semantics to the
|
|
52
|
+
* array-based path so the decision is stable regardless of caller.
|
|
56
53
|
*/
|
|
57
54
|
export function splitForRow(row, holdouts = defaultHoldouts()) {
|
|
58
55
|
const region = row.components.region;
|
|
@@ -64,12 +61,12 @@ export function splitForRow(row, holdouts = defaultHoldouts()) {
|
|
|
64
61
|
return hashBucket(row.source_id, 2) === 0 ? "val" : "test";
|
|
65
62
|
}
|
|
66
63
|
/**
|
|
67
|
-
* Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are
|
|
68
|
-
*
|
|
64
|
+
* Compute a `SplitManifest` from an iterable of labeled (or canonical) rows. Both shapes are accepted — only
|
|
65
|
+
* `source_id`, `country`, `corpus_version`, and `components.region` are consulted.
|
|
69
66
|
*
|
|
70
|
-
* Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via
|
|
71
|
-
*
|
|
72
|
-
*
|
|
67
|
+
* Retained for in-memory callers (tests; small-scale fixture runs). Real-data builds via `buildCorpus` use the
|
|
68
|
+
* streaming path (`splitForRow` + `writeSplitManifestsFromLabeledFiles`) to avoid materializing every aligned row's
|
|
69
|
+
* split membership in heap.
|
|
73
70
|
*/
|
|
74
71
|
export function splitRows(rows, opts = {}) {
|
|
75
72
|
const holdouts = opts.holdouts ?? defaultHoldouts();
|
|
@@ -106,10 +103,9 @@ export function hashBucket(id, n) {
|
|
|
106
103
|
return u % n;
|
|
107
104
|
}
|
|
108
105
|
/**
|
|
109
|
-
* Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated
|
|
110
|
-
*
|
|
111
|
-
*
|
|
112
|
-
* corpus version.
|
|
106
|
+
* Write a `SplitManifest` to `<outputDir>/{train,val,test}.json`. The manifests are line-separated source_id lists (one
|
|
107
|
+
* id per line) so they diff cleanly in git. Also writes `<outputDir>/MANIFEST.json` with the full structured manifest
|
|
108
|
+
* including holdouts + counts + corpus version.
|
|
113
109
|
*
|
|
114
110
|
* Reruns produce byte-identical files (the underlying `splitRows` is deterministic).
|
|
115
111
|
*/
|
|
@@ -127,14 +123,12 @@ export async function writeSplitManifests(manifest, outputDir) {
|
|
|
127
123
|
await writeFile(join(outputDir, "SPLIT_MANIFEST.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
|
|
128
124
|
}
|
|
129
125
|
/**
|
|
130
|
-
* Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests +
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
* files that exceed in-memory thresholds.
|
|
126
|
+
* Streaming variant of `writeSplitManifests`: derives the per-split source-id .txt manifests + `SPLIT_MANIFEST.json` by
|
|
127
|
+
* streaming three per-split labeled-row JSONL files (one per split). Memory cost is O(1) — `sort(1)` from coreutils
|
|
128
|
+
* handles the deterministic sort with disk spill for files that exceed in-memory thresholds.
|
|
134
129
|
*
|
|
135
|
-
* Used by `buildCorpus` after the align loop has already partitioned labeled rows into
|
|
136
|
-
*
|
|
137
|
-
* passed in (zero re-scan).
|
|
130
|
+
* Used by `buildCorpus` after the align loop has already partitioned labeled rows into `labeled-{train,val,test}.jsonl`
|
|
131
|
+
* via `splitForRow`. Counts are pre-computed by the align loop and passed in (zero re-scan).
|
|
138
132
|
*/
|
|
139
133
|
export async function writeSplitManifestsFromLabeledFiles(opts) {
|
|
140
134
|
await mkdir(opts.outputDir, { recursive: true });
|
|
@@ -154,9 +148,8 @@ export async function writeSplitManifestsFromLabeledFiles(opts) {
|
|
|
154
148
|
return summary.counts;
|
|
155
149
|
}
|
|
156
150
|
/**
|
|
157
|
-
* Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input →
|
|
158
|
-
*
|
|
159
|
-
* stays O(1) regardless of labeled-row count.
|
|
151
|
+
* Extract `source_id`s from a labeled JSONL file, write them sorted to `outPath`. Empty input → empty output file (not
|
|
152
|
+
* absent). Uses `sort(1)` for disk-spilling external sort so peak memory stays O(1) regardless of labeled-row count.
|
|
160
153
|
*/
|
|
161
154
|
async function streamSortedSourceIds(labeledJsonlPath, outPath) {
|
|
162
155
|
const unsortedPath = `${outPath}.unsorted`;
|
package/out/src/split.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;
|
|
1
|
+
{"version":3,"file":"split.js","sourceRoot":"","sources":["../../src/split.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAA;AAC1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC7D,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC3D,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AA4B/C;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe;IAC9B,OAAO;QACN,EAAE,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,CAAC;QAC5D,EAAE,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC;QAC3C,EAAE,EAAE,CAAC,UAAU,EAAE,IAAI,EAAE,wBAAwB,EAAE,IAAI,CAAC;KACtD,CAAA;AACF,CAAC;AAID;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAC1B,GAAgE,EAChE,WAA8C,eAAe,EAAE;IAE/D,MAAM,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,MAAM,CAAA;IACpC,MAAM,eAAe,GAAG,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IACnD,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IAE1E,IAAI,CAAC,SAAS;QAAE,OAAO,OAAO,CAAA;IAE9B,oFAAoF;IACpF,OAAO,UAAU,CAAC,GAAG,CAAC,SAAS,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAA;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,SAAS,CAAC,IAA6B,EAAE,OAAqB,EAAE;IAC/E,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IACnD,MAAM,KAAK,GAAa,EAAE,CAAA;IAC1B,MAAM,GAAG,GAAa,EAAE,CAAA;IACxB,MAAM,IAAI,GAAa,EAAE,CAAA;IACzB,IAAI,cAAc,GAAG,EAAE,CAAA;IAEvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,cAAc,IAAI,GAAG,CAAC,cAAc;YAAE,cAAc,GAAG,GAAG,CAAC,cAAc,CAAA;QAC9E,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;QAExC,IAAI,KAAK,KAAK,OAAO;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;aAC3C,IAAI,KAAK,KAAK,KAAK;YAAE,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;;YAC5C,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IAC9B,CAAC;IAED,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;IAErD,OAAO;QACN,KAAK;QACL,GAAG;QACH,IAAI;QACJ,QAAQ;QACR,cAAc;QACd,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KAC1E,CAAA;AACF,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,UAAU,CAAC,EAAU,EAAE,CAAS;IAC/C,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAA;IACvD,mDAAmD;IACnD,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,aAAa,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,OAAO,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;IAElG,OAAO,CAAC,GAAG,CAAC,CAAA;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,QAAuB,EAAE,SAAiB;IACnF,MAAM,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAE3C,KAAK,MAAM,IAAI,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACtD,MAAM,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QACzC,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IACzG,CAAC;IACD,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,QAAQ,CAAC,cAAc;QACvC,QAAQ,EAAE,QAAQ,CAAC,QAAQ;QAC3B,MAAM,EAAE,QAAQ,CAAC,MAAM;KACvB,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;AACzG,CAAC;AAKD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,mCAAmC,CAAC,IAMzD;IACA,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAChD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,eAAe,EAAE,CAAA;IAEnD,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAA;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,KAAK,MAAM,CAAC,CAAA;QACpD,MAAM,qBAAqB,CAAC,WAAW,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACpE,MAAM,OAAO,GAAG;QACf,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,QAAQ;QACR,MAAM,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE;KACjC,CAAA;IACD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,qBAAqB,CAAC,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;IAE7G,OAAO,OAAO,CAAC,MAAM,CAAA;AACtB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,qBAAqB,CAAC,gBAAwB,EAAE,OAAe;IAC7E,MAAM,YAAY,GAAG,GAAG,OAAO,WAAW,CAAA;IAC1C,MAAM,GAAG,GAAG,iBAAiB,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IACjE,MAAM,EAAE,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,gBAAgB,CAAC,gBAAgB,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC,CAAA;IAEpH,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,EAAE,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACtB,IAAI,CAAC,IAAI;gBAAE,OAAM;YAEjB,IAAI,CAAC;gBACJ,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAA2B,CAAA;gBAEtD,IAAI,OAAO,GAAG,CAAC,SAAS,KAAK,QAAQ;oBAAE,GAAG,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,SAAS,IAAI,CAAC,CAAA;YACvE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,MAAM,CAAC,GAAY,CAAC,CAAA;YACrB,CAAC;QACF,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YACnB,GAAG,CAAC,GAAG,EAAE,CAAA;QACV,CAAC,CAAC,CAAA;QACF,EAAE,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACtB,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAA;QAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;IACxB,CAAC,CAAC,CAAA;IAEF,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,wEAAwE;QACxE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,YAAY,EAAE,IAAI,EAAE,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAA;QACnG,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;QACxB,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACxB,IAAI,IAAI,KAAK,CAAC;gBAAE,OAAO,EAAE,CAAA;;gBACpB,MAAM,CAAC,IAAI,KAAK,CAAC,yBAAyB,IAAI,EAAE,CAAC,CAAC,CAAA;QACxD,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IACF,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC3C,CAAC"}
|
|
@@ -40,9 +40,8 @@ export interface AnchorAbsorptionSynthesisOpts {
|
|
|
40
40
|
random?: () => number;
|
|
41
41
|
forceTemplate?: AnchorAbsorptionTemplate;
|
|
42
42
|
/**
|
|
43
|
-
* Real US ZIPs (in the anchor lookup) to use as the LEADING 5-digit house number — so the painted
|
|
44
|
-
*
|
|
45
|
-
* pilot-anchor-lookup.json.
|
|
43
|
+
* Real US ZIPs (in the anchor lookup) to use as the LEADING 5-digit house number — so the painted anchor fires on it,
|
|
44
|
+
* the CASE-H/anchor-fp trigger. Builder loads these from pilot-anchor-lookup.json.
|
|
46
45
|
*/
|
|
47
46
|
realZips?: ReadonlyArray<string>;
|
|
48
47
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-anchor-absorption.d.ts","sourceRoot":"","sources":["../../src/synthesize-anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEzD,MAAM,WAAW,yBAAyB;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,MAAM,wBAAwB,GACjC,eAAe,GACf,wBAAwB,GACxB,YAAY,GACZ,MAAM,GACN,WAAW,GACX,cAAc,GACd,UAAU,CAAA;AAEb,MAAM,WAAW,6BAA6B;IAC7C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,aAAa,CAAC,EAAE,wBAAwB,CAAA;IACxC
|
|
1
|
+
{"version":3,"file":"synthesize-anchor-absorption.d.ts","sourceRoot":"","sources":["../../src/synthesize-anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAEzD,MAAM,WAAW,yBAAyB;IACzC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;CAChB;AAED,MAAM,MAAM,wBAAwB,GACjC,eAAe,GACf,wBAAwB,GACxB,YAAY,GACZ,MAAM,GACN,WAAW,GACX,cAAc,GACd,UAAU,CAAA;AAEb,MAAM,WAAW,6BAA6B;IAC7C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,aAAa,CAAC,EAAE,wBAAwB,CAAA;IACxC;;;OAGG;IACH,QAAQ,CAAC,EAAE,aAAa,CAAC,MAAM,CAAC,CAAA;CAChC;AAED,MAAM,WAAW,8BAA8B;IAC9C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC,CAAA;IACjD,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,wBAAwB,CAAA;CAClC;AAgED,4DAA4D;AAC5D,wBAAgB,6BAA6B,CAC5C,IAAI,GAAE,6BAAkC,GACtC,8BAA8B,CA8GhC;AAaD,eAAO,MAAM,aAAa,EAAE,aAAa,CAAC,wBAAwB,CAQjE,CAAA"}
|
|
@@ -33,8 +33,8 @@ function pick(arr, random) {
|
|
|
33
33
|
return arr[Math.floor(random() * arr.length)];
|
|
34
34
|
}
|
|
35
35
|
/**
|
|
36
|
-
* A realistic US house number: mostly 1–4 digits (the common range), 25% a real 5-digit ZIP (the
|
|
37
|
-
*
|
|
36
|
+
* A realistic US house number: mostly 1–4 digits (the common range), 25% a real 5-digit ZIP (the HARD case — a 5-digit
|
|
37
|
+
* leading number that is still a house number when a locality is present).
|
|
38
38
|
*/
|
|
39
39
|
function houseNum(random, realZips) {
|
|
40
40
|
if (random() < 0.25)
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-anchor-absorption.js","sourceRoot":"","sources":["../../src/synthesize-anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;
|
|
1
|
+
{"version":3,"file":"synthesize-anchor-absorption.js","sourceRoot":"","sources":["../../src/synthesize-anchor-absorption.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAoCH,SAAS,IAAI,CAAI,GAAqB,EAAE,MAAoB;IAC3D,OAAO,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAE,CAAA;AAC/C,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,MAAoB,EAAE,QAA+B;IACtE,IAAI,MAAM,EAAE,GAAG,IAAI;QAAE,OAAO,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;IAElD,OAAO,MAAM,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAA;AAC/C,CAAC;AAED,wGAAwG;AACxG,0EAA0E;AAC1E,MAAM,YAAY,GAAG;IACpB,MAAM;IACN,KAAK;IACL,KAAK;IACL,OAAO;IACP,OAAO;IACP,MAAM;IACN,YAAY;IACZ,SAAS;IACT,MAAM;IACN,MAAM;IACN,cAAc;IACd,UAAU;IACV,QAAQ;IACR,YAAY;IACZ,WAAW;IACX,OAAO;IACP,UAAU;IACV,iBAAiB;CACjB,CAAA;AACD,MAAM,YAAY,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAA;AAC1F,MAAM,SAAS,GAA6C;IAC3D,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC5D,EAAE,QAAQ,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IACzD,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IACxD,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IACvD,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IACvD,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC5D,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;IAC1D,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE;CAC3D,CAAA;AACD,wGAAwG;AACxG,mEAAmE;AACnE,MAAM,aAAa,GAAG,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAA;AAChE,yEAAyE;AACzE,MAAM,SAAS,GAAG;IACjB,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,aAAa,EAAE;IAChE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,mBAAmB,EAAE;IACvE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,kBAAkB,EAAE;IACtE,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE;IAC9D,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE;CACnE,CAAA;AACD,MAAM,UAAU,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,CAAC,CAAA;AAC/D,qGAAqG;AACrG,MAAM,SAAS,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAA;AAE/D,4DAA4D;AAC5D,MAAM,UAAU,6BAA6B,CAC5C,OAAsC,EAAE;IAExC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAA;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAA;IACzG,MAAM,QAAQ,GAAG,IAAI,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,IAAI,IAAI,CAAC,YAAY,EAAE,MAAM,CAAC,EAAE,CAAA;IAE5E,IAAI,QAAQ,KAAK,eAAe,EAAE,CAAC;QAClC,oGAAoG;QACpG,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAClC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;QAExE,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;YACvG,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC/B,uGAAuG;QACvG,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAClC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAA;QAC1C,MAAM,GAAG,GAAG,GAAG,GAAG,IAAI,MAAM,KAAK,MAAM,EAAE,CAAA;QAEzC,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE;YAC7C,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,MAAM,EAAE,CAAC;QACzB,sFAAsF;QACtF,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAA;QACnC,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,EAAE,EAAE,CAAA;QAE5D,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,YAAY,EAAE,EAAE,EAAE;YAC9F,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;QAC9B,mGAAmG;QACnG,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACpC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;QAEzE,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;YACxG,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,cAAc,EAAE,CAAC;QACjC,gGAAgG;QAChG,iGAAiG;QACjG,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAA;QAElC,IAAI,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;YACpB,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,IAAI,MAAM,EAAE,EAAE,UAAU,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAA;QACzG,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QAEjC,OAAO;YACN,GAAG,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC,QAAQ,EAAE;YAC3B,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;YACnD,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IAED,IAAI,QAAQ,KAAK,wBAAwB,EAAE,CAAC;QAC3C,oGAAoG;QACpG,mGAAmG;QACnG,oGAAoG;QACpG,gGAAgG;QAChG,mGAAmG;QACnG,uGAAuG;QACvG,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QACrC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QACtE,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,MAAM,EAAE,CAAA;QAEvD,OAAO;YACN,GAAG;YACH,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE;YACtE,MAAM,EAAE,OAAO;YACf,QAAQ;SACR,CAAA;IACF,CAAC;IACD,sGAAsG;IACtG,MAAM,EAAE,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,MAAM,GAAG,GAAG,GAAG,EAAE,IAAI,MAAM,KAAK,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEvE,OAAO;QACN,GAAG;QACH,UAAU,EAAE,EAAE,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE;QACtG,MAAM,EAAE,OAAO;QACf,QAAQ;KACR,CAAA;AACF,CAAC;AAED,gGAAgG;AAChG,iEAAiE;AACjE,EAAE;AACF,oGAAoG;AACpG,yGAAyG;AACzG,4GAA4G;AAC5G,uGAAuG;AACvG,0GAA0G;AAC1G,yGAAyG;AACzG,yGAAyG;AACzG,0GAA0G;AAC1G,MAAM,CAAC,MAAM,aAAa,GAA4C;IACrE,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC;IAC5D,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC;IACrE,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC;IACzD,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC;IACnD,GAAG,KAAK,CAA2B,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;IACvD,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC;IAC3D,GAAG,KAAK,CAA2B,EAAE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;CACvD,CAAA"}
|
|
@@ -68,10 +68,9 @@ export interface SynthesizedBoundaryStressRow {
|
|
|
68
68
|
template: BoundaryStressTemplate;
|
|
69
69
|
}
|
|
70
70
|
/**
|
|
71
|
-
* Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple
|
|
72
|
-
*
|
|
73
|
-
*
|
|
74
|
-
* `alignRow` locates + BIO-labels it.
|
|
71
|
+
* Synthesize one boundary-stress row. `base` is optional — when omitted, a locale-appropriate tuple is drawn from the
|
|
72
|
+
* internal pools (so the generator is self-contained; a build script can pass real tuples for scale + diversity). Every
|
|
73
|
+
* component value is a verbatim substring of `raw`, so `alignRow` locates + BIO-labels it.
|
|
75
74
|
*/
|
|
76
75
|
export declare function synthesizeBoundaryStressRow(base: BoundaryStressBaseTuple | undefined, opts?: BoundaryStressSynthesisOpts): SynthesizedBoundaryStressRow;
|
|
77
76
|
//# sourceMappingURL=synthesize-boundary-stress.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"synthesize-boundary-stress.d.ts","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC/B,mBAAmB,GACnB,uBAAuB,GACvB,WAAW,GACX,2BAA2B,GAG3B,eAAe,GACf,4BAA4B,CAAA;AAE/B,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,mEAAmE;IACnE,aAAa,CAAC,EAAE,sBAAsB,CAAA;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,sBAAsB,CAAA;CAChC;AAyRD
|
|
1
|
+
{"version":3,"file":"synthesize-boundary-stress.d.ts","sourceRoot":"","sources":["../../src/synthesize-boundary-stress.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiDG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAE9C,MAAM,MAAM,sBAAsB,GAC/B,mBAAmB,GACnB,uBAAuB,GACvB,WAAW,GACX,2BAA2B,GAG3B,eAAe,GACf,4BAA4B,CAAA;AAE/B,MAAM,WAAW,uBAAuB;IACvC,QAAQ,EAAE,MAAM,CAAA;IAChB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,OAAO,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,2BAA2B;IAC3C,MAAM,CAAC,EAAE,MAAM,MAAM,CAAA;IACrB,mEAAmE;IACnE,aAAa,CAAC,EAAE,sBAAsB,CAAA;CACtC;AAED,MAAM,WAAW,4BAA4B;IAC5C,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,YAAY,CAAC,YAAY,CAAC,CAAA;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,sBAAsB,CAAA;CAChC;AAyRD;;;;GAIG;AACH,wBAAgB,2BAA2B,CAC1C,IAAI,EAAE,uBAAuB,GAAG,SAAS,EACzC,IAAI,GAAE,2BAAgC,GACpC,4BAA4B,CAqI9B"}
|