@mailwoman/corpus 8.6.0 → 9.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/out/src/adapter.d.ts +41 -0
- package/out/src/adapter.d.ts.map +1 -1
- package/out/src/adapter.js +46 -1
- package/out/src/adapter.js.map +1 -1
- package/out/src/adapters/ban/adapter.d.ts +4 -3
- package/out/src/adapters/ban/adapter.d.ts.map +1 -1
- package/out/src/adapters/ban/adapter.js +60 -67
- package/out/src/adapters/ban/adapter.js.map +1 -1
- package/out/src/adapters/ban/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/ban/street-decompose.js +18 -25
- package/out/src/adapters/ban/street-decompose.js.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +0 -6
- package/out/src/adapters/fcc-bdc/adapter.d.ts.map +1 -1
- package/out/src/adapters/fcc-bdc/adapter.js +2 -24
- package/out/src/adapters/fcc-bdc/adapter.js.map +1 -1
- package/out/src/adapters/geonames/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames/adapter.js +70 -76
- package/out/src/adapters/geonames/adapter.js.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts.map +1 -1
- package/out/src/adapters/geonames-postal/adapter.js +44 -49
- package/out/src/adapters/geonames-postal/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts.map +1 -1
- package/out/src/adapters/gnaf/adapter.js +4 -7
- package/out/src/adapters/gnaf/adapter.js.map +1 -1
- package/out/src/adapters/gnaf/assemble.d.ts.map +1 -1
- package/out/src/adapters/gnaf/assemble.js +6 -11
- package/out/src/adapters/gnaf/assemble.js.map +1 -1
- package/out/src/adapters/openaddresses/adapter.d.ts.map +1 -1
- package/out/src/adapters/openaddresses/adapter.js +2 -7
- package/out/src/adapters/openaddresses/adapter.js.map +1 -1
- package/out/src/adapters/overture/adapter.d.ts.map +1 -1
- package/out/src/adapters/overture/adapter.js +3 -7
- package/out/src/adapters/overture/adapter.js.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-hi-schools/adapter.js +55 -73
- package/out/src/adapters/state-hi-schools/adapter.js.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.js +58 -76
- package/out/src/adapters/state-ia-contractors/adapter.js.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.js +67 -85
- package/out/src/adapters/state-ny-notaries/adapter.js.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts.map +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.js +69 -87
- package/out/src/adapters/state-tx-notaries/adapter.js.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts.map +1 -1
- package/out/src/adapters/synth-po-box/adapter.js +7 -15
- package/out/src/adapters/synth-po-box/adapter.js.map +1 -1
- package/out/src/adapters/tiger/street-decompose.d.ts.map +1 -1
- package/out/src/adapters/tiger/street-decompose.js +18 -27
- package/out/src/adapters/tiger/street-decompose.js.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +3 -3
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js +62 -80
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.js.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.js +60 -82
- package/out/src/adapters/usgov-imls-pls/adapter.js.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.js +58 -65
- package/out/src/adapters/usgov-irs-bmf/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nad/adapter.js +5 -8
- package/out/src/adapters/usgov-nad/adapter.js.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-nppes/adapter.js +58 -76
- package/out/src/adapters/usgov-nppes/adapter.js.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts.map +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js +51 -71
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.js.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts.map +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.js +24 -6
- package/out/src/adapters/wof-admin-jp/adapter.js.map +1 -1
- package/out/src/build.d.ts.map +1 -1
- package/out/src/build.js +2 -1
- package/out/src/build.js.map +1 -1
- package/out/src/golden.d.ts +5 -0
- package/out/src/golden.d.ts.map +1 -1
- package/out/src/golden.js +23 -9
- package/out/src/golden.js.map +1 -1
- package/out/src/parquet-wrapper/reader.d.ts +8 -0
- package/out/src/parquet-wrapper/reader.d.ts.map +1 -1
- package/out/src/parquet-wrapper/reader.js +16 -0
- package/out/src/parquet-wrapper/reader.js.map +1 -1
- package/out/src/parquet-wrapper/schema.d.ts +4 -0
- package/out/src/parquet-wrapper/schema.d.ts.map +1 -1
- package/out/src/parquet-wrapper/schema.js.map +1 -1
- package/out/src/parquet.d.ts.map +1 -1
- package/out/src/parquet.js +2 -13
- package/out/src/parquet.js.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.d.ts.map +1 -1
- package/out/src/shard-recipes/anchor-absorption.js +2 -1
- package/out/src/shard-recipes/anchor-absorption.js.map +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts.map +1 -1
- package/out/src/shard-recipes/country-balanced.js +68 -60
- package/out/src/shard-recipes/country-balanced.js.map +1 -1
- package/out/src/shard-recipes/fr-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-fragment.js +8 -5
- package/out/src/shard-recipes/fr-fragment.js.map +1 -1
- package/out/src/shard-recipes/fr-order.d.ts.map +1 -1
- package/out/src/shard-recipes/fr-order.js +14 -58
- package/out/src/shard-recipes/fr-order.js.map +1 -1
- package/out/src/shard-recipes/german.d.ts.map +1 -1
- package/out/src/shard-recipes/german.js +11 -58
- package/out/src/shard-recipes/german.js.map +1 -1
- package/out/src/shard-recipes/index.d.ts.map +1 -1
- package/out/src/shard-recipes/index.js +2 -0
- package/out/src/shard-recipes/index.js.map +1 -1
- package/out/src/shard-recipes/intersection.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts.map +1 -1
- package/out/src/shard-recipes/intersection.js +24 -62
- package/out/src/shard-recipes/intersection.js.map +1 -1
- package/out/src/shard-recipes/locale.d.ts.map +1 -1
- package/out/src/shard-recipes/locale.js +14 -16
- package/out/src/shard-recipes/locale.js.map +1 -1
- package/out/src/shard-recipes/no-fragment.d.ts.map +1 -1
- package/out/src/shard-recipes/no-fragment.js +8 -5
- package/out/src/shard-recipes/no-fragment.js.map +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts.map +1 -1
- package/out/src/shard-recipes/no-street-led.js +8 -5
- package/out/src/shard-recipes/no-street-led.js.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +3 -3
- package/out/src/shard-recipes/po-box-cedex.d.ts.map +1 -1
- package/out/src/shard-recipes/po-box-cedex.js +51 -94
- package/out/src/shard-recipes/po-box-cedex.js.map +1 -1
- package/out/src/shard-recipes/scaffold.d.ts +59 -9
- package/out/src/shard-recipes/scaffold.d.ts.map +1 -1
- package/out/src/shard-recipes/scaffold.js +51 -30
- package/out/src/shard-recipes/scaffold.js.map +1 -1
- package/out/src/shard-recipes/street-affix.d.ts.map +1 -1
- package/out/src/shard-recipes/street-affix.js +56 -82
- package/out/src/shard-recipes/street-affix.js.map +1 -1
- package/out/src/shard-recipes/sub-venue-sources.d.ts +231 -0
- package/out/src/shard-recipes/sub-venue-sources.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue-sources.js +669 -0
- package/out/src/shard-recipes/sub-venue-sources.js.map +1 -0
- package/out/src/shard-recipes/sub-venue.d.ts +269 -0
- package/out/src/shard-recipes/sub-venue.d.ts.map +1 -0
- package/out/src/shard-recipes/sub-venue.js +709 -0
- package/out/src/shard-recipes/sub-venue.js.map +1 -0
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts.map +1 -1
- package/out/src/shard-recipes/unit.js +22 -64
- package/out/src/shard-recipes/unit.js.map +1 -1
- package/out/src/split.d.ts +7 -0
- package/out/src/split.d.ts.map +1 -1
- package/out/src/split.js +7 -0
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize.d.ts.map +1 -1
- package/out/src/synthesize.js +1 -12
- package/out/src/synthesize.js.map +1 -1
- package/out/src/tools/align-shard.d.ts.map +1 -1
- package/out/src/tools/align-shard.js +3 -7
- package/out/src/tools/align-shard.js.map +1 -1
- package/out/src/tools/audit.d.ts.map +1 -1
- package/out/src/tools/audit.js +4 -4
- package/out/src/tools/audit.js.map +1 -1
- package/out/src/tools/corpus-stats.d.ts +2 -2
- package/out/src/tools/corpus-stats.d.ts.map +1 -1
- package/out/src/tools/corpus-stats.js +18 -31
- package/out/src/tools/corpus-stats.js.map +1 -1
- package/out/src/tools/fetch/download.d.ts.map +1 -1
- package/out/src/tools/fetch/download.js +5 -6
- package/out/src/tools/fetch/download.js.map +1 -1
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -1
- package/out/src/tools/fetch/imls-pls.js +2 -2
- package/out/src/tools/fetch/imls-pls.js.map +1 -1
- package/out/src/tools/fetch/index.d.ts +14 -1
- package/out/src/tools/fetch/index.d.ts.map +1 -1
- package/out/src/tools/fetch/index.js +14 -1
- package/out/src/tools/fetch/index.js.map +1 -1
- package/out/src/tools/fetch/nad.d.ts +1 -1
- package/out/src/tools/fetch/nad.js +1 -1
- package/out/src/tools/fetch/nppes.d.ts.map +1 -1
- package/out/src/tools/fetch/nppes.js +2 -1
- package/out/src/tools/fetch/nppes.js.map +1 -1
- package/out/src/tools/fetch/openaddresses.d.ts +1 -1
- package/out/src/tools/fetch/openaddresses.js +1 -1
- package/out/src/tools/fetch/ourairports.d.ts +45 -0
- package/out/src/tools/fetch/ourairports.d.ts.map +1 -0
- package/out/src/tools/fetch/ourairports.js +123 -0
- package/out/src/tools/fetch/ourairports.js.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts +171 -0
- package/out/src/tools/fetch/wikidata-subvenue.d.ts.map +1 -0
- package/out/src/tools/fetch/wikidata-subvenue.js +275 -0
- package/out/src/tools/fetch/wikidata-subvenue.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -1
- package/out/src/tools/golden-expand.js +18 -16
- package/out/src/tools/golden-expand.js.map +1 -1
- package/out/src/tools/golden-promote.d.ts.map +1 -1
- package/out/src/tools/golden-promote.js +12 -6
- package/out/src/tools/golden-promote.js.map +1 -1
- package/out/src/tools/golden-relabel-street.d.ts +196 -0
- package/out/src/tools/golden-relabel-street.d.ts.map +1 -0
- package/out/src/tools/golden-relabel-street.js +513 -0
- package/out/src/tools/golden-relabel-street.js.map +1 -0
- package/out/src/tools/index.d.ts +4 -0
- package/out/src/tools/index.d.ts.map +1 -1
- package/out/src/tools/index.js +4 -0
- package/out/src/tools/index.js.map +1 -1
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -1
- package/out/src/tools/jsonl-to-parquet.js +3 -2
- package/out/src/tools/jsonl-to-parquet.js.map +1 -1
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -1
- package/out/src/tools/lint-shard-vocab.js +3 -2
- package/out/src/tools/lint-shard-vocab.js.map +1 -1
- package/out/src/tools/lint-shard.d.ts +3 -3
- package/out/src/tools/lint-shard.d.ts.map +1 -1
- package/out/src/tools/lint-shard.js +23 -32
- package/out/src/tools/lint-shard.js.map +1 -1
- package/out/src/tools/overlay-manifest.d.ts.map +1 -1
- package/out/src/tools/overlay-manifest.js +2 -1
- package/out/src/tools/overlay-manifest.js.map +1 -1
- package/out/src/tools/overture-subvenue.d.ts +112 -0
- package/out/src/tools/overture-subvenue.d.ts.map +1 -0
- package/out/src/tools/overture-subvenue.js +144 -0
- package/out/src/tools/overture-subvenue.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +1 -1
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -1
- package/out/src/tools/shard-kryptonite.js +5 -4
- package/out/src/tools/shard-kryptonite.js.map +1 -1
- package/out/src/tools/shard-translit.d.ts +6 -3
- package/out/src/tools/shard-translit.d.ts.map +1 -1
- package/out/src/tools/shard-translit.js +8 -6
- package/out/src/tools/shard-translit.js.map +1 -1
- package/out/src/tools/sub-venue-lexicon.d.ts +507 -0
- package/out/src/tools/sub-venue-lexicon.d.ts.map +1 -0
- package/out/src/tools/sub-venue-lexicon.js +817 -0
- package/out/src/tools/sub-venue-lexicon.js.map +1 -0
- package/out/src/tools/sub-venue-promotions.d.ts +94 -0
- package/out/src/tools/sub-venue-promotions.d.ts.map +1 -0
- package/out/src/tools/sub-venue-promotions.js +266 -0
- package/out/src/tools/sub-venue-promotions.js.map +1 -0
- package/out/src/wof-json.d.ts +2 -2
- package/out/src/wof-json.d.ts.map +1 -1
- package/out/src/wof-json.js +5 -8
- package/out/src/wof-json.js.map +1 -1
- package/package.json +8 -8
- package/src/adapter.ts +58 -1
- package/src/adapters/ban/adapter.ts +55 -65
- package/src/adapters/ban/street-decompose.ts +17 -25
- package/src/adapters/fcc-bdc/adapter.ts +2 -33
- package/src/adapters/geonames/adapter.ts +64 -72
- package/src/adapters/geonames-postal/adapter.ts +42 -50
- package/src/adapters/gnaf/adapter.ts +4 -7
- package/src/adapters/gnaf/assemble.ts +6 -10
- package/src/adapters/openaddresses/adapter.ts +2 -7
- package/src/adapters/overture/adapter.ts +3 -6
- package/src/adapters/state-hi-schools/adapter.ts +50 -73
- package/src/adapters/state-ia-contractors/adapter.ts +52 -76
- package/src/adapters/state-ny-notaries/adapter.ts +61 -85
- package/src/adapters/state-tx-notaries/adapter.ts +60 -84
- package/src/adapters/synth-po-box/adapter.ts +7 -17
- package/src/adapters/tiger/street-decompose.ts +21 -31
- package/src/adapters/usgov-hrsa-fqhc/adapter.ts +61 -84
- package/src/adapters/usgov-imls-pls/adapter.ts +54 -82
- package/src/adapters/usgov-irs-bmf/adapter.ts +53 -63
- package/src/adapters/usgov-nad/adapter.ts +5 -8
- package/src/adapters/usgov-nppes/adapter.ts +51 -75
- package/src/adapters/usgov-samhsa-treatment-locator/adapter.ts +45 -71
- package/src/adapters/wof-admin-jp/adapter.ts +34 -8
- package/src/build.ts +2 -1
- package/src/golden.ts +24 -9
- package/src/parquet-wrapper/reader.ts +19 -0
- package/src/parquet-wrapper/schema.ts +4 -0
- package/src/parquet.ts +3 -16
- package/src/shard-recipes/anchor-absorption.ts +2 -1
- package/src/shard-recipes/country-balanced.ts +69 -70
- package/src/shard-recipes/fr-fragment.ts +10 -7
- package/src/shard-recipes/fr-order.ts +13 -63
- package/src/shard-recipes/german.ts +12 -65
- package/src/shard-recipes/index.ts +2 -0
- package/src/shard-recipes/intersection.ts +32 -60
- package/src/shard-recipes/locale.ts +14 -16
- package/src/shard-recipes/no-fragment.ts +10 -7
- package/src/shard-recipes/no-street-led.ts +10 -7
- package/src/shard-recipes/po-box-cedex.ts +68 -119
- package/src/shard-recipes/scaffold.ts +82 -28
- package/src/shard-recipes/street-affix.ts +58 -95
- package/src/shard-recipes/sub-venue-sources.ts +895 -0
- package/src/shard-recipes/sub-venue.ts +982 -0
- package/src/shard-recipes/unit.ts +22 -70
- package/src/split.ts +7 -0
- package/src/synthesize.ts +1 -15
- package/src/tools/align-shard.ts +3 -6
- package/src/tools/audit.ts +6 -5
- package/src/tools/corpus-stats.ts +25 -33
- package/src/tools/fetch/download.ts +7 -5
- package/src/tools/fetch/imls-pls.ts +2 -2
- package/src/tools/fetch/index.ts +14 -1
- package/src/tools/fetch/nad.ts +1 -1
- package/src/tools/fetch/nppes.ts +2 -1
- package/src/tools/fetch/openaddresses.ts +1 -1
- package/src/tools/fetch/ourairports.ts +166 -0
- package/src/tools/fetch/wikidata-subvenue.ts +386 -0
- package/src/tools/golden-expand.ts +18 -13
- package/src/tools/golden-promote.ts +15 -6
- package/src/tools/golden-relabel-street.ts +748 -0
- package/src/tools/index.ts +4 -0
- package/src/tools/jsonl-to-parquet.ts +3 -2
- package/src/tools/lint-shard-vocab.ts +3 -2
- package/src/tools/lint-shard.ts +33 -36
- package/src/tools/overlay-manifest.ts +2 -1
- package/src/tools/overture-subvenue.ts +215 -0
- package/src/tools/shard-kryptonite.ts +5 -4
- package/src/tools/shard-translit.ts +12 -7
- package/src/tools/sub-venue-lexicon.ts +1250 -0
- package/src/tools/sub-venue-promotions.ts +330 -0
- package/src/wof-json.ts +5 -8
|
@@ -32,7 +32,8 @@
|
|
|
32
32
|
*/
|
|
33
33
|
import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
|
|
34
34
|
import { join } from "node:path";
|
|
35
|
-
import {
|
|
35
|
+
import { sha256File } from "@mailwoman/core/utils";
|
|
36
|
+
import { createNewlineWriter, JSONSpliterator } from "spliterator";
|
|
36
37
|
// ── Types ──────────────────────────────────────────────────────────────────
|
|
37
38
|
/**
|
|
38
39
|
* Components a candidate needs before it is rich enough to be worth a golden-set slot.
|
|
@@ -101,10 +102,10 @@ function isSuspicious(entry) {
|
|
|
101
102
|
/**
|
|
102
103
|
* Read a JSONL, tolerating a missing file (returns `[]`) — prior golden dirs may lack a bucket.
|
|
103
104
|
*/
|
|
104
|
-
function readJSONLIfPresent(path) {
|
|
105
|
+
async function readJSONLIfPresent(path) {
|
|
105
106
|
if (!existsSync(path))
|
|
106
107
|
return [];
|
|
107
|
-
return
|
|
108
|
+
return Array.fromAsync(JSONSpliterator.fromAsync(path));
|
|
108
109
|
}
|
|
109
110
|
// ── Main ───────────────────────────────────────────────────────────────────
|
|
110
111
|
export async function promoteGolden(options, report) {
|
|
@@ -113,7 +114,7 @@ export async function promoteGolden(options, report) {
|
|
|
113
114
|
const applyFilters = !options.noFilters;
|
|
114
115
|
const dryRun = options.dryRun ?? false;
|
|
115
116
|
report?.(`reading candidates: ${options.input}`);
|
|
116
|
-
const candidates = readJSONLIfPresent(options.input);
|
|
117
|
+
const candidates = await readJSONLIfPresent(options.input);
|
|
117
118
|
report?.(` ${candidates.length} candidates loaded`);
|
|
118
119
|
// Forward-copy base: existing entries from the prior golden version go forward verbatim,
|
|
119
120
|
// and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
|
|
@@ -123,7 +124,7 @@ export async function promoteGolden(options, report) {
|
|
|
123
124
|
if (existsSync(priorDir)) {
|
|
124
125
|
for (const f of readdirSync(priorDir).filter((n) => n.endsWith(".jsonl"))) {
|
|
125
126
|
const country = f.replace(".jsonl", "").toUpperCase();
|
|
126
|
-
const entries = readJSONLIfPresent(join(priorDir, f));
|
|
127
|
+
const entries = await readJSONLIfPresent(join(priorDir, f));
|
|
127
128
|
priorEntries.push({ country, entries });
|
|
128
129
|
for (const e of entries) {
|
|
129
130
|
seenNormalized.add(normalize(e.raw));
|
|
@@ -221,7 +222,12 @@ export async function promoteGolden(options, report) {
|
|
|
221
222
|
for (const [key, entries] of buckets) {
|
|
222
223
|
const filename = `${key.toLowerCase()}.jsonl`;
|
|
223
224
|
const path = join(outDir, filename);
|
|
224
|
-
|
|
225
|
+
{
|
|
226
|
+
await using out = createNewlineWriter(path);
|
|
227
|
+
for (const entry of entries) {
|
|
228
|
+
await out.write(JSON.stringify(entry));
|
|
229
|
+
}
|
|
230
|
+
}
|
|
225
231
|
manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) };
|
|
226
232
|
}
|
|
227
233
|
// Forward-copy non-.jsonl files (README.md, etc.) from prior
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"golden-promote.js","sourceRoot":"","sources":["../../../src/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACzF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"golden-promote.js","sourceRoot":"","sources":["../../../src/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,WAAW,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AACzF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAClD,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAElE,8EAA8E;AAE9E;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B;;GAEG;AACH,MAAM,yBAAyB,GAAG,CAAC,CAAA;AAEnC;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAgD5B,6EAA6E;AAE7E,SAAS,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACtD,CAAC;AAED;;;GAGG;AACH,SAAS,iBAAiB,CAAC,KAAkB;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;IAE3D,IAAI,cAAc,GAAG,qBAAqB;QAAE,OAAO,KAAK,CAAA;IACxD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE5D,OAAO,UAAU,GAAG,CAAC,CAAA;AACtB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,KAAkB;IACjD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,yBAAyB;QAAE,OAAO,KAAK,CAAA;IAElF,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAA;IAE1C,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE/C,IAAI,WAAW,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAEpC,OAAO,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAkB;IACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAA;IAErB,4CAA4C;IAC5C,IAAI,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IACxD,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE1D,IAAI,YAAY,KAAK,aAAa;QAAE,OAAO,IAAI,CAAA;IAE/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,kBAAkB;QAAE,OAAO,IAAI,CAAA;IAE3D,uBAAuB;IACvB,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAE9E;;GAEG;AACH,KAAK,UAAU,kBAAkB,CAAI,IAAY;IAChD,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC;QAAE,OAAO,EAAE,CAAA;IAEhC,OAAO,KAAK,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAI,IAAI,CAAC,CAAC,CAAA;AAC3D,CAAC;AAED,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAA;IACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAA;IAC3D,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,SAAS,CAAA;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,EAAE,CAAC,uBAAuB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAChD,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAc,OAAO,CAAC,KAAK,CAAC,CAAA;IACvE,MAAM,EAAE,CAAC,KAAK,UAAU,CAAC,MAAM,oBAAoB,CAAC,CAAA;IAEpD,yFAAyF;IACzF,oFAAoF;IACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;IACxC,MAAM,YAAY,GAAkD,EAAE,CAAA;IACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IAExC,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC3E,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;YACrD,MAAM,OAAO,GAAG,MAAM,kBAAkB,CAAc,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAA;YACxE,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;YAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzB,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YACrC,CAAC;YAED,MAAM,EAAE,CAAC,WAAW,OAAO,KAAK,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAC9E,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,iBAAiB,QAAQ,6BAA6B,CAAC,CAAA;IACjE,CAAC;IAED,cAAc;IACd,MAAM,KAAK,GAAiB;QAC3B,YAAY,EAAE,UAAU,CAAC,MAAM;QAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QACzF,IAAI,EAAE,CAAC;QACP,UAAU,EAAE,EAAE;KACd,CAAA;IAED,MAAM,QAAQ,GAAkB,EAAE,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IAErC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEhC,+CAA+C;QAC/C,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,mCAAmC;QACnC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAA;YAE7B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;gBAEzB,SAAQ;YACT,CAAC;YAED,IAAI,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAA;gBAEnC,SAAQ;YACT,CAAC;YAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;gBAE9B,SAAQ;YACT,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnB,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAErB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,OAAO,CAAA;QACvC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,uDAAuD;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAA;IAEhD,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,YAAY,EAAE,CAAC;QACjD,8FAA8F;QAC9F,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;QAEnD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QACrB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,SAAS;IACT,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACxB,MAAM,EAAE,CAAC,eAAe,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE9D,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,KAAK,GAAG,CAAC,WAAW,EAAE,WAAW,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yBAAyB,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;IACpE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC,CAAA;IAC9E,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,CAAA;IACxE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,qBAAqB,CAAC,CAAA;IAE/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,6BAA6B,CAAC,CAAA;QAEvC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEtC,MAAM,QAAQ,GAKV;QACH,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,IAAI,EAAE,OAAO,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAC5C,KAAK,EAAE,EAAE;KACT,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QAEnC,CAAC;YACA,YAAY,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;YAE3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;YACvC,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAA;IACvF,CAAC;IAED,6DAA6D;IAC7D,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1B,KAAK,MAAM,CAAC,IAAI,WAAW,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YAC5E,YAAY,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;YAChD,MAAM,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC;IACF,CAAC;IAED,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAA;IACtF,MAAM,EAAE,CAAC,iBAAiB,MAAM,EAAE,CAAC,CAAA;IAEnC,OAAO,KAAK,CAAA;AACb,CAAC"}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Golden-set street-suffix relabel — v0.1.2 → v0.1.3.
|
|
7
|
+
*
|
|
8
|
+
* ## Why this exists
|
|
9
|
+
*
|
|
10
|
+
* The golden answer key and the training corpus disagreed about ONE thing, and the v9.0.0
|
|
11
|
+
* promotion gate read the disagreement as a model regression (`us.street` 87.4 vs a floor of
|
|
12
|
+
* 87.8). The corpus SPLITS a US street into `street` + `street_suffix` — TIGER's adapter
|
|
13
|
+
* decomposes at `corpus/src/adapters/tiger/street-decompose.ts`, the `street-affix` shard recipe
|
|
14
|
+
* teaches it from USPS Pub-28, and `ComponentTag` carries `street_suffix` as a first-class tag.
|
|
15
|
+
* The golden set FOLDED it: 2,216 US rows carry a `street`, and exactly 2 of them label a
|
|
16
|
+
* `street_suffix`. Operator ruling, 2026-08-06: **the split is canonical**; the golden is the
|
|
17
|
+
* stale side. This tool moves the answer key onto the corpus convention.
|
|
18
|
+
*
|
|
19
|
+
* ## The instrument
|
|
20
|
+
*
|
|
21
|
+
* `matchTrailingSuffix` from `@mailwoman/codex/us` — the USPS Pub-28 Appendix C table, which is
|
|
22
|
+
* also what the corpus shard recipe splits on. The table is NOT re-implemented here, and the
|
|
23
|
+
* libpostal dictionary TIGER reads is deliberately not used: measured on this golden set the two
|
|
24
|
+
* disagree on 51 US rows, and the disagreements run in the codex table's favour (libpostal's
|
|
25
|
+
* `directionals.txt` lists `center|c`, so TIGER reads the `C` of "C STREET" as a directional
|
|
26
|
+
* prefix and then emits no suffix at all).
|
|
27
|
+
*
|
|
28
|
+
* ## What it changes, and what it refuses to
|
|
29
|
+
*
|
|
30
|
+
* Applied only to rows whose `country` is `US`. Three branches, mirroring the SHAPE of TIGER's
|
|
31
|
+
* `decomposeStreet` on codex tables:
|
|
32
|
+
*
|
|
33
|
+
* - **street type** — the last whitespace-separated word is a Pub-28 suffix, and something is left
|
|
34
|
+
* over: "Main St" → `street: "Main"`, `street_suffix: "St"` (1,559 rows).
|
|
35
|
+
* - **street type + post-directional** — the last word is a directional AND the one before it is a
|
|
36
|
+
* Pub-28 suffix: "Pennsylvania Avenue NW" → `street: "Pennsylvania"`,
|
|
37
|
+
* `street_suffix: "Avenue NW"` (347 rows). The post-directional joins the suffix rather than
|
|
38
|
+
* becoming a tag of its own, because that is what the corpus adapter emits; there is no
|
|
39
|
+
* `street_postfix` tag to move it to.
|
|
40
|
+
* - **everything else is left folded** and reported. In particular a BARE post-directional tail
|
|
41
|
+
* ("Seymour East", "BROADWAY N" — 16 rows) is NOT split: a directional is not a Pub-28 suffix,
|
|
42
|
+
* and the observed rows in that class are unit-contaminated ("1ST AVE SW BOX E", where the
|
|
43
|
+
* trailing "E" is a box letter).
|
|
44
|
+
*
|
|
45
|
+
* FR rows are untouched, deliberately and permanently as far as this tool is concerned. French
|
|
46
|
+
* street typology puts the type FIRST ("Rue de la Paix") and the golden labels only 7 of 665 FR
|
|
47
|
+
* street rows with a `street_prefix`; whether FR should split at all is a different question with
|
|
48
|
+
* a different table behind it, and nothing here should be read as having answered it.
|
|
49
|
+
*
|
|
50
|
+
* ## Surface bytes
|
|
51
|
+
*
|
|
52
|
+
* The split is a cut at a whitespace run in the ORIGINAL string — no trimming, no case
|
|
53
|
+
* normalization, no re-joining of tokens. `street + gap + street_suffix` reconstructs the input
|
|
54
|
+
* byte-for-byte, so the whitespace between them belongs to neither span (the same shape the corpus
|
|
55
|
+
* adapter's spans have). The tool asserts this per row and refuses to write a file if it ever
|
|
56
|
+
* fails.
|
|
57
|
+
*/
|
|
58
|
+
/**
|
|
59
|
+
* A golden-set row, as stored one-per-line in `us.jsonl` / `fr.jsonl` / `adversarial.jsonl`. Only the fields this tool
|
|
60
|
+
* reads are modeled; every other key rides through untouched.
|
|
61
|
+
*/
|
|
62
|
+
export interface GoldenStreetRow {
|
|
63
|
+
raw: string;
|
|
64
|
+
components: Record<string, string>;
|
|
65
|
+
country?: string;
|
|
66
|
+
source?: string;
|
|
67
|
+
notes?: string;
|
|
68
|
+
[key: string]: unknown;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* What the tool decided about one row. Every value except the two `split-*` classes means "left folded".
|
|
72
|
+
*/
|
|
73
|
+
export type GoldenRelabelClass = "split-suffix" | "split-suffix-postdirectional" | "split-prefix-only" | "already-split" | "single-token" | "suffix-only-street" | "postdirectional-tail-only" | "no-suffix-match" | "no-street" | "not-us" | "untrimmed-street";
|
|
74
|
+
/**
|
|
75
|
+
* A review trigger on a row the tool DID change. A flag is never an adjudication — it marks the row for the operator's
|
|
76
|
+
* deck, and the split is applied either way.
|
|
77
|
+
*/
|
|
78
|
+
export interface GoldenRelabelFlag {
|
|
79
|
+
kind: "name-prone-suffix" | "venue-context" | "remainder-is-affix";
|
|
80
|
+
detail: string;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Row-level relabel outcome.
|
|
84
|
+
*/
|
|
85
|
+
export interface GoldenRelabelResult {
|
|
86
|
+
/**
|
|
87
|
+
* The row to write. Identical object reference when nothing changed.
|
|
88
|
+
*/
|
|
89
|
+
row: GoldenStreetRow;
|
|
90
|
+
changed: boolean;
|
|
91
|
+
rowClass: GoldenRelabelClass;
|
|
92
|
+
flags: GoldenRelabelFlag[];
|
|
93
|
+
/**
|
|
94
|
+
* A leading directional was lifted into `street_prefix` on this row.
|
|
95
|
+
*/
|
|
96
|
+
prefixSplit: boolean;
|
|
97
|
+
/**
|
|
98
|
+
* The street span as it stood in the parent version — recorded for the review deck.
|
|
99
|
+
*/
|
|
100
|
+
beforeStreet?: string;
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Options for {@linkcode relabelGoldenStreetRow}.
|
|
104
|
+
*/
|
|
105
|
+
export interface RelabelStreetRowOptions {
|
|
106
|
+
/**
|
|
107
|
+
* Also lift a folded LEADING directional out into `street_prefix`. Default true.
|
|
108
|
+
*
|
|
109
|
+
* On by default because the fold cuts both ways and the answer key has to be corrected on both, or the correction is
|
|
110
|
+
* not a correction: 207 of the 1,682 split dev rows (12.3%) still opened with a directional after the suffix move —
|
|
111
|
+
* "N Desmet Avenue" would have graded `street: "N Desmet"` against a model that says `street_prefix: "N", street:
|
|
112
|
+
* "Desmet"`. Turn it OFF only to measure what the prefix fold alone costs.
|
|
113
|
+
*/
|
|
114
|
+
splitPrefix?: boolean;
|
|
115
|
+
}
|
|
116
|
+
/**
|
|
117
|
+
* Decide, and apply, the US street-span split for ONE golden row. Pure: never mutates its argument, and returns the
|
|
118
|
+
* same object reference when the row is left alone.
|
|
119
|
+
*/
|
|
120
|
+
export declare function relabelGoldenStreetRow(row: GoldenStreetRow, options?: RelabelStreetRowOptions): GoldenRelabelResult;
|
|
121
|
+
/**
|
|
122
|
+
* Per-class row counts for one relabelled file.
|
|
123
|
+
*/
|
|
124
|
+
export type GoldenRelabelCounts = Record<GoldenRelabelClass, number>;
|
|
125
|
+
/**
|
|
126
|
+
* One line of the review deck: what a changed (or notably unchanged) row looked like before and after.
|
|
127
|
+
*/
|
|
128
|
+
export interface GoldenRelabelDeckEntry {
|
|
129
|
+
file: string;
|
|
130
|
+
line: number;
|
|
131
|
+
raw: string;
|
|
132
|
+
rowClass: GoldenRelabelClass;
|
|
133
|
+
before: Record<string, string>;
|
|
134
|
+
after: Record<string, string>;
|
|
135
|
+
flags: GoldenRelabelFlag[];
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Options for {@linkcode relabelGoldenDirectory}.
|
|
139
|
+
*/
|
|
140
|
+
export interface RelabelGoldenOptions {
|
|
141
|
+
/**
|
|
142
|
+
* Parent golden version dir (read-only), e.g. `data/eval/golden/v0.1.2`.
|
|
143
|
+
*/
|
|
144
|
+
input: string;
|
|
145
|
+
/**
|
|
146
|
+
* Output golden version dir. Created; never overwritten in place.
|
|
147
|
+
*/
|
|
148
|
+
output: string;
|
|
149
|
+
/**
|
|
150
|
+
* Review-deck JSONL path. Default `<output>/REVIEW-DECK.jsonl`.
|
|
151
|
+
*/
|
|
152
|
+
deck?: string;
|
|
153
|
+
/**
|
|
154
|
+
* Parent version label recorded in the manifest. Default: the input dir's basename.
|
|
155
|
+
*/
|
|
156
|
+
parentLabel?: string;
|
|
157
|
+
/**
|
|
158
|
+
* Tool provenance recorded in the manifest — the commit the relabel ran at.
|
|
159
|
+
*/
|
|
160
|
+
commit?: string;
|
|
161
|
+
/**
|
|
162
|
+
* Passed through to {@linkcode relabelGoldenStreetRow}. Default true.
|
|
163
|
+
*/
|
|
164
|
+
splitPrefix?: boolean;
|
|
165
|
+
}
|
|
166
|
+
/**
|
|
167
|
+
* Aggregate outcome for a whole golden version.
|
|
168
|
+
*/
|
|
169
|
+
export interface RelabelGoldenReport {
|
|
170
|
+
files: Record<string, {
|
|
171
|
+
entries: number;
|
|
172
|
+
changed: number;
|
|
173
|
+
flagged: number;
|
|
174
|
+
prefixSplit: number;
|
|
175
|
+
counts: GoldenRelabelCounts;
|
|
176
|
+
}>;
|
|
177
|
+
deckPath: string;
|
|
178
|
+
outputDir: string;
|
|
179
|
+
totalChanged: number;
|
|
180
|
+
totalFlagged: number;
|
|
181
|
+
}
|
|
182
|
+
/**
|
|
183
|
+
* Relabel every `.jsonl` in a golden version dir, writing a new version dir plus a review deck and a MANIFEST that
|
|
184
|
+
* records the convention, the parent, and the counts. Non-JSONL siblings (README, split manifests) are copied forward
|
|
185
|
+
* so the new version is self-contained; nested split dirs (`dev/`, `test/`) are relabelled recursively.
|
|
186
|
+
*/
|
|
187
|
+
export declare function relabelGoldenDirectory(options: RelabelGoldenOptions, report?: (line: string) => void): Promise<RelabelGoldenReport>;
|
|
188
|
+
/**
|
|
189
|
+
* Every relabel class that means the row was left folded, for callers that want to report the residue.
|
|
190
|
+
*/
|
|
191
|
+
export declare function isLeftFolded(rowClass: GoldenRelabelClass): boolean;
|
|
192
|
+
/**
|
|
193
|
+
* True when a golden dir declares the US-split convention — i.e. it is safe to grade it with an UNFOLDED scorer.
|
|
194
|
+
*/
|
|
195
|
+
export declare function goldenDeclaresSplitStreets(dir: string): boolean;
|
|
196
|
+
//# sourceMappingURL=golden-relabel-street.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"golden-relabel-street.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-relabel-street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAYH;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC3B,cAAc,GACd,8BAA8B,GAC9B,mBAAmB,GACnB,eAAe,GACf,cAAc,GACd,oBAAoB,GACpB,2BAA2B,GAC3B,iBAAiB,GACjB,WAAW,GACX,QAAQ,GACR,kBAAkB,CAAA;AAErB;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,mBAAmB,GAAG,eAAe,GAAG,oBAAoB,CAAA;IAClE,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,GAAG,EAAE,eAAe,CAAA;IACpB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B;;OAEG;IACH,WAAW,EAAE,OAAO,CAAA;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAmJD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACrC,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,uBAA4B,GACnC,mBAAmB,CAmIrB;AAQD;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;AAEpE;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B,KAAK,EAAE,iBAAiB,EAAE,CAAA;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CACZ,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,mBAAmB,CAAA;KAAE,CACvG,CAAA;IACD,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;CACpB;AA0BD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,oBAAoB,EAC7B,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkB,GAC1C,OAAO,CAAC,mBAAmB,CAAC,CAgJ9B;AAsDD;;GAEG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAElE;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAY/D"}
|