@mailwoman/corpus 5.8.0 → 5.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/out/src/adapter.d.ts +1 -1
- package/out/src/adapters/ban/adapter.d.ts +1 -1
- package/out/src/adapters/fcc-bdc/adapter.d.ts +1 -1
- package/out/src/adapters/geonames/adapter.d.ts +1 -1
- package/out/src/adapters/geonames-postal/adapter.d.ts +1 -1
- package/out/src/adapters/gnaf/adapter.d.ts +1 -1
- package/out/src/adapters/index.d.ts +19 -19
- package/out/src/adapters/openaddresses/adapter.d.ts +1 -1
- package/out/src/adapters/overture/adapter.d.ts +1 -1
- package/out/src/adapters/state-hi-schools/adapter.d.ts +1 -1
- package/out/src/adapters/state-ia-contractors/adapter.d.ts +1 -1
- package/out/src/adapters/state-ny-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/state-tx-notaries/adapter.d.ts +1 -1
- package/out/src/adapters/synth-po-box/adapter.d.ts +2 -2
- package/out/src/adapters/tiger/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-hrsa-fqhc/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-imls-pls/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-irs-bmf/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-nad/adapter.d.ts +2 -2
- package/out/src/adapters/usgov-nad/adapter.js +1 -1
- package/out/src/adapters/usgov-nppes/adapter.d.ts +1 -1
- package/out/src/adapters/usgov-samhsa-treatment-locator/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-jp/adapter.d.ts +1 -1
- package/out/src/adapters/wof-admin-json/adapter.d.ts +2 -2
- package/out/src/adapters/wof-postalcode-json/adapter.d.ts +2 -2
- package/out/src/align.d.ts +2 -2
- package/out/src/build.d.ts +4 -4
- package/out/src/index.d.ts +22 -21
- package/out/src/index.d.ts.map +1 -1
- package/out/src/index.js +1 -0
- package/out/src/index.js.map +1 -1
- package/out/src/parquet-wrapper/index.d.ts +3 -3
- package/out/src/parquet-wrapper/reader.d.ts +1 -1
- package/out/src/parquet-wrapper/writer.d.ts +4 -2
- package/out/src/parquet-wrapper/writer.d.ts.map +1 -1
- package/out/src/parquet-wrapper/writer.js +3 -1
- package/out/src/parquet-wrapper/writer.js.map +1 -1
- package/out/src/parquet.d.ts +3 -3
- package/out/src/runner.d.ts +2 -2
- package/out/src/shard-recipes/anchor-absorption.d.ts +1 -1
- package/out/src/shard-recipes/boundary-stress.d.ts +1 -1
- package/out/src/shard-recipes/country-balanced.d.ts +1 -1
- package/out/src/shard-recipes/cz-pcfirst-preposition.d.ts +1 -1
- package/out/src/shard-recipes/fr-admin-split.d.ts +1 -1
- package/out/src/shard-recipes/fr-bare-street.d.ts +1 -1
- package/out/src/shard-recipes/fr-order.d.ts +1 -1
- package/out/src/shard-recipes/german.d.ts +1 -1
- package/out/src/shard-recipes/house-venue.d.ts +1 -1
- package/out/src/shard-recipes/index.d.ts +2 -2
- package/out/src/shard-recipes/intersection.d.ts +1 -1
- package/out/src/shard-recipes/locale.d.ts +2 -2
- package/out/src/shard-recipes/nl-postcode.d.ts +1 -1
- package/out/src/shard-recipes/no-street-led.d.ts +1 -1
- package/out/src/shard-recipes/no-street.d.ts +1 -1
- package/out/src/shard-recipes/po-box-cedex.d.ts +1 -1
- package/out/src/shard-recipes/po-box.d.ts +1 -1
- package/out/src/shard-recipes/si-bare-village.d.ts +1 -1
- package/out/src/shard-recipes/street-affix.d.ts +1 -1
- package/out/src/shard-recipes/street-bare.d.ts +1 -1
- package/out/src/shard-recipes/street.d.ts +1 -1
- package/out/src/shard-recipes/unit.d.ts +1 -1
- package/out/src/split.d.ts +1 -1
- package/out/src/split.js +1 -1
- package/out/src/split.js.map +1 -1
- package/out/src/synthesize-boundary-stress.d.ts +1 -1
- package/out/src/synthesize-german.d.ts +1 -1
- package/out/src/synthesize-house-venue.d.ts +1 -1
- package/out/src/synthesize-intersection.d.ts +1 -1
- package/out/src/synthesize-no-street.d.ts +1 -1
- package/out/src/synthesize-po-box.d.ts +1 -1
- package/out/src/synthesize-street.d.ts +1 -1
- package/out/src/synthesize.d.ts +2 -2
- package/out/src/tools/align-shard.d.ts +8 -0
- package/out/src/tools/align-shard.d.ts.map +1 -0
- package/out/src/tools/align-shard.js +58 -0
- package/out/src/tools/align-shard.js.map +1 -0
- package/out/src/tools/audit.d.ts +28 -0
- package/out/src/tools/audit.d.ts.map +1 -0
- package/out/src/tools/audit.js +273 -0
- package/out/src/tools/audit.js.map +1 -0
- package/out/src/tools/corpus-stats.d.ts +44 -0
- package/out/src/tools/corpus-stats.d.ts.map +1 -0
- package/out/src/tools/corpus-stats.js +151 -0
- package/out/src/tools/corpus-stats.js.map +1 -0
- package/out/src/tools/fetch/ban.d.ts +31 -0
- package/out/src/tools/fetch/ban.d.ts.map +1 -0
- package/out/src/tools/fetch/ban.js +248 -0
- package/out/src/tools/fetch/ban.js.map +1 -0
- package/out/src/tools/fetch/download.d.ts +49 -0
- package/out/src/tools/fetch/download.d.ts.map +1 -0
- package/out/src/tools/fetch/download.js +81 -0
- package/out/src/tools/fetch/download.js.map +1 -0
- package/out/src/tools/fetch/hrsa.d.ts +18 -0
- package/out/src/tools/fetch/hrsa.d.ts.map +1 -0
- package/out/src/tools/fetch/hrsa.js +46 -0
- package/out/src/tools/fetch/hrsa.js.map +1 -0
- package/out/src/tools/fetch/imls-pls.d.ts +25 -0
- package/out/src/tools/fetch/imls-pls.d.ts.map +1 -0
- package/out/src/tools/fetch/imls-pls.js +120 -0
- package/out/src/tools/fetch/imls-pls.js.map +1 -0
- package/out/src/tools/fetch/index.d.ts +118 -0
- package/out/src/tools/fetch/index.d.ts.map +1 -0
- package/out/src/tools/fetch/index.js +117 -0
- package/out/src/tools/fetch/index.js.map +1 -0
- package/out/src/tools/fetch/nad.d.ts +56 -0
- package/out/src/tools/fetch/nad.d.ts.map +1 -0
- package/out/src/tools/fetch/nad.js +225 -0
- package/out/src/tools/fetch/nad.js.map +1 -0
- package/out/src/tools/fetch/nppes.d.ts +26 -0
- package/out/src/tools/fetch/nppes.d.ts.map +1 -0
- package/out/src/tools/fetch/nppes.js +136 -0
- package/out/src/tools/fetch/nppes.js.map +1 -0
- package/out/src/tools/fetch/openaddresses.d.ts +59 -0
- package/out/src/tools/fetch/openaddresses.d.ts.map +1 -0
- package/out/src/tools/fetch/openaddresses.js +307 -0
- package/out/src/tools/fetch/openaddresses.js.map +1 -0
- package/out/src/tools/fetch/state-hi-schools.d.ts +25 -0
- package/out/src/tools/fetch/state-hi-schools.d.ts.map +1 -0
- package/out/src/tools/fetch/state-hi-schools.js +171 -0
- package/out/src/tools/fetch/state-hi-schools.js.map +1 -0
- package/out/src/tools/fetch/state-sources.d.ts +22 -0
- package/out/src/tools/fetch/state-sources.d.ts.map +1 -0
- package/out/src/tools/fetch/state-sources.js +110 -0
- package/out/src/tools/fetch/state-sources.js.map +1 -0
- package/out/src/tools/fetch/tiger-full.d.ts +36 -0
- package/out/src/tools/fetch/tiger-full.d.ts.map +1 -0
- package/out/src/tools/fetch/tiger-full.js +239 -0
- package/out/src/tools/fetch/tiger-full.js.map +1 -0
- package/out/src/tools/golden-expand.d.ts +74 -0
- package/out/src/tools/golden-expand.d.ts.map +1 -0
- package/out/src/tools/golden-expand.js +394 -0
- package/out/src/tools/golden-expand.js.map +1 -0
- package/out/src/tools/golden-promote.d.ts +60 -0
- package/out/src/tools/golden-promote.d.ts.map +1 -0
- package/out/src/tools/golden-promote.js +223 -0
- package/out/src/tools/golden-promote.js.map +1 -0
- package/out/src/tools/index.d.ts +24 -0
- package/out/src/tools/index.d.ts.map +1 -0
- package/out/src/tools/index.js +24 -0
- package/out/src/tools/index.js.map +1 -0
- package/out/src/tools/ingest-csv.d.ts +45 -0
- package/out/src/tools/ingest-csv.d.ts.map +1 -0
- package/out/src/tools/ingest-csv.js +309 -0
- package/out/src/tools/ingest-csv.js.map +1 -0
- package/out/src/tools/jsonl-to-parquet.d.ts +49 -0
- package/out/src/tools/jsonl-to-parquet.d.ts.map +1 -0
- package/out/src/tools/jsonl-to-parquet.js +157 -0
- package/out/src/tools/jsonl-to-parquet.js.map +1 -0
- package/out/src/tools/lint-shard-vocab.d.ts +66 -0
- package/out/src/tools/lint-shard-vocab.d.ts.map +1 -0
- package/out/src/tools/lint-shard-vocab.js +266 -0
- package/out/src/tools/lint-shard-vocab.js.map +1 -0
- package/out/src/tools/lint-shard.d.ts +73 -0
- package/out/src/tools/lint-shard.d.ts.map +1 -0
- package/out/src/tools/lint-shard.js +350 -0
- package/out/src/tools/lint-shard.js.map +1 -0
- package/out/src/tools/overlay-manifest.d.ts +35 -0
- package/out/src/tools/overlay-manifest.d.ts.map +1 -0
- package/out/src/tools/overlay-manifest.js +85 -0
- package/out/src/tools/overlay-manifest.js.map +1 -0
- package/out/src/tools/shard-kryptonite.d.ts +28 -0
- package/out/src/tools/shard-kryptonite.d.ts.map +1 -0
- package/out/src/tools/shard-kryptonite.js +101 -0
- package/out/src/tools/shard-kryptonite.js.map +1 -0
- package/out/src/tools/shard-translit.d.ts +42 -0
- package/out/src/tools/shard-translit.d.ts.map +1 -0
- package/out/src/tools/shard-translit.js +196 -0
- package/out/src/tools/shard-translit.js.map +1 -0
- package/package.json +63 -7
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Generate golden-set candidate entries by LLM-driven surface-form synthesis from a verified-label
|
|
7
|
+
* seed pulled out of the corpus's labeled test split.
|
|
8
|
+
*
|
|
9
|
+
* ## Why this approach
|
|
10
|
+
*
|
|
11
|
+
* The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
|
|
12
|
+
* doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
|
|
13
|
+
* would be unverified.
|
|
14
|
+
*
|
|
15
|
+
* This module takes the middle path:
|
|
16
|
+
*
|
|
17
|
+
* 1. **Seeds come from corpus-v0.2.0 test shard** — already through the alignment pipeline, so labels
|
|
18
|
+
* are pipeline-verified.
|
|
19
|
+
* 2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
|
|
20
|
+
* component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
|
|
21
|
+
* 3. **Programmatic validator drops hallucinations** — every component value must appear as a
|
|
22
|
+
* substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
|
|
23
|
+
* silently; cost is wasted tokens, never bad-labeled golden entries.
|
|
24
|
+
*
|
|
25
|
+
* ## Usage
|
|
26
|
+
*
|
|
27
|
+
* ```sh
|
|
28
|
+
* DEEPSEEK_API_KEY=sk-... \
|
|
29
|
+
* mailwoman corpus golden expand \
|
|
30
|
+
* --count 1000 \
|
|
31
|
+
* --variants 5 \
|
|
32
|
+
* --output data/eval/golden/candidates/expand-$(date +%Y%m%d-%H%M%S).jsonl
|
|
33
|
+
* ```
|
|
34
|
+
*
|
|
35
|
+
* ## Env
|
|
36
|
+
*
|
|
37
|
+
* - `DEEPSEEK_API_KEY` — required for provider `deepseek`
|
|
38
|
+
* - `ANTHROPIC_API_KEY` — required for provider `anthropic`
|
|
39
|
+
*
|
|
40
|
+
* ## What this module does NOT do
|
|
41
|
+
*
|
|
42
|
+
* - Does not commit anything or modify the versioned golden dir. Candidates land in
|
|
43
|
+
* `data/eval/golden/candidates/` for operator review (skim, prune, then run
|
|
44
|
+
* `mailwoman corpus golden promote`).
|
|
45
|
+
* - Does not score the LLM's quality — that's an eyeball job after pilot lands.
|
|
46
|
+
* - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
|
|
47
|
+
*/
|
|
48
|
+
import { mkdir } from "node:fs/promises";
|
|
49
|
+
import { dirname } from "node:path";
|
|
50
|
+
import { ParquetReader } from "@dsnp/parquetjs";
|
|
51
|
+
import { $private } from "@mailwoman/core/env";
|
|
52
|
+
import { dataRootPath, writeJSONL } from "@mailwoman/core/utils";
|
|
53
|
+
// ── Seed loading ──────────────────────────────────────────────────────────
|
|
54
|
+
/**
|
|
55
|
+
* Decode BIO labels + tokens into a verified components map. Mirrors the Python `decode_components` in
|
|
56
|
+
* mailwoman_train/eval.py — first-occurrence-wins per tag, contiguous B-X/I-X runs concatenated with a single space
|
|
57
|
+
* (the canonical separator used by corpus alignment).
|
|
58
|
+
*/
|
|
59
|
+
function decodeComponents(tokens, labels) {
|
|
60
|
+
const out = {};
|
|
61
|
+
let currentTag = null;
|
|
62
|
+
let currentTokens = [];
|
|
63
|
+
const flush = () => {
|
|
64
|
+
if (currentTag && currentTokens.length > 0 && !(currentTag in out)) {
|
|
65
|
+
out[currentTag] = currentTokens.join(" ").trim();
|
|
66
|
+
}
|
|
67
|
+
currentTag = null;
|
|
68
|
+
currentTokens = [];
|
|
69
|
+
};
|
|
70
|
+
for (let i = 0; i < labels.length; i++) {
|
|
71
|
+
const label = labels[i];
|
|
72
|
+
const tok = tokens[i] ?? "";
|
|
73
|
+
if (label === "O") {
|
|
74
|
+
flush();
|
|
75
|
+
continue;
|
|
76
|
+
}
|
|
77
|
+
const [prefix, tag] = label.split("-", 2);
|
|
78
|
+
if (prefix === "B" || currentTag !== tag) {
|
|
79
|
+
flush();
|
|
80
|
+
currentTag = tag;
|
|
81
|
+
currentTokens = [tok];
|
|
82
|
+
}
|
|
83
|
+
else {
|
|
84
|
+
currentTokens.push(tok);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
flush();
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
async function loadSeeds(corpusPath, count, includeSources, report) {
|
|
91
|
+
const paths = corpusPath
|
|
92
|
+
.split(",")
|
|
93
|
+
.map((p) => p.trim())
|
|
94
|
+
.filter(Boolean);
|
|
95
|
+
report?.(`reading seeds from ${paths.length} shard(s) (target: ${count}, stratified)`);
|
|
96
|
+
if (includeSources) {
|
|
97
|
+
report?.(` include-sources filter: ${Array.from(includeSources).join(", ")}`);
|
|
98
|
+
}
|
|
99
|
+
// Stratified sampling: read all rows from all shards, group by source. Bounded by per-source
|
|
100
|
+
// reservoir: keep at most max(2*count, 5000) rows per source so we don't blow memory on train
|
|
101
|
+
// shards (1M rows × many shards). Sampling later is uniform within each pool.
|
|
102
|
+
const bySource = new Map();
|
|
103
|
+
const PER_SOURCE_CAP = Math.max(2 * count, 5000);
|
|
104
|
+
let scanned = 0;
|
|
105
|
+
let skippedThinComponents = 0;
|
|
106
|
+
for (const path of paths) {
|
|
107
|
+
const reader = await ParquetReader.openFile(path);
|
|
108
|
+
const cursor = reader.getCursor();
|
|
109
|
+
while (true) {
|
|
110
|
+
const row = (await cursor.next());
|
|
111
|
+
if (!row)
|
|
112
|
+
break;
|
|
113
|
+
scanned++;
|
|
114
|
+
// Source allow-list (--include-sources) — applied early to skip parsing rows we won't use
|
|
115
|
+
if (includeSources && !includeSources.has(row.source))
|
|
116
|
+
continue;
|
|
117
|
+
const components = decodeComponents(row.tokens ?? [], row.labels ?? []);
|
|
118
|
+
// Skip rows with too few components — single-name wof-admin entries don't make useful seeds
|
|
119
|
+
if (Object.keys(components).length < 2) {
|
|
120
|
+
skippedThinComponents++;
|
|
121
|
+
continue;
|
|
122
|
+
}
|
|
123
|
+
const seed = {
|
|
124
|
+
raw: row.raw,
|
|
125
|
+
components,
|
|
126
|
+
country: row.country,
|
|
127
|
+
source: row.source,
|
|
128
|
+
source_id: row.source_id,
|
|
129
|
+
};
|
|
130
|
+
let bucket = bySource.get(row.source);
|
|
131
|
+
if (!bucket) {
|
|
132
|
+
bucket = [];
|
|
133
|
+
bySource.set(row.source, bucket);
|
|
134
|
+
}
|
|
135
|
+
if (bucket.length < PER_SOURCE_CAP) {
|
|
136
|
+
bucket.push(seed);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
await reader.close();
|
|
140
|
+
}
|
|
141
|
+
report?.(` scanned ${scanned} rows across ${paths.length} shard(s); thin-components dropped: ${skippedThinComponents}`);
|
|
142
|
+
report?.(` per-source pool sizes:`);
|
|
143
|
+
for (const [src, pool] of bySource) {
|
|
144
|
+
report?.(` ${src}: ${pool.length}`);
|
|
145
|
+
}
|
|
146
|
+
// Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
|
|
147
|
+
// to sources in alphabetical order. If a pool is smaller than its target, take all of it.
|
|
148
|
+
const sources = Array.from(bySource.keys()).sort();
|
|
149
|
+
const perSource = Math.floor(count / sources.length);
|
|
150
|
+
const remainder = count - perSource * sources.length;
|
|
151
|
+
const picked = [];
|
|
152
|
+
for (let i = 0; i < sources.length; i++) {
|
|
153
|
+
const src = sources[i];
|
|
154
|
+
const pool = bySource.get(src);
|
|
155
|
+
const target = perSource + (i < remainder ? 1 : 0);
|
|
156
|
+
// Random subsample without replacement — deterministic via shuffle then slice
|
|
157
|
+
for (let j = pool.length - 1; j > 0; j--) {
|
|
158
|
+
const k = Math.floor(Math.random() * (j + 1));
|
|
159
|
+
[pool[j], pool[k]] = [pool[k], pool[j]];
|
|
160
|
+
}
|
|
161
|
+
const take = Math.min(target, pool.length);
|
|
162
|
+
picked.push(...pool.slice(0, take));
|
|
163
|
+
if (take < target) {
|
|
164
|
+
report?.(` ⚠ ${src}: requested ${target}, pool had ${pool.length}`);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
report?.(` → loaded ${picked.length} seeds across ${sources.length} sources`);
|
|
168
|
+
return picked;
|
|
169
|
+
}
|
|
170
|
+
const SYSTEM_PROMPT = `You are a postal-address surface-form generator. Given a structured address, produce realistic variants a human might type into a geocoder.
|
|
171
|
+
|
|
172
|
+
CONSTRAINT — you MUST preserve every kept component value verbatim in the output. You may:
|
|
173
|
+
- vary case (UPPER, lower, Title Case)
|
|
174
|
+
- abbreviate (Saint → St, Avenue → Ave, Boulevard → Blvd, North → N, etc.)
|
|
175
|
+
- vary punctuation (commas, dashes, spaces, line breaks)
|
|
176
|
+
- reorder components (postcode-first, country-first, address-only)
|
|
177
|
+
- drop OPTIONAL components — list them in "dropped"
|
|
178
|
+
|
|
179
|
+
OPTIONAL components (allowed to drop): country, postcode, dependent_locality, subregion, cedex.
|
|
180
|
+
REQUIRED components (must keep, even if input has them): locality, region (when present),
|
|
181
|
+
street, house_number, venue (when present).
|
|
182
|
+
|
|
183
|
+
ALWAYS keep AT LEAST 2 components in the final raw text. Single-component variants like
|
|
184
|
+
"VT" or "Paris" are USELESS as eval entries — do not produce them.
|
|
185
|
+
|
|
186
|
+
You MUST NOT:
|
|
187
|
+
- introduce typos (validator drops these silently — wasted tokens)
|
|
188
|
+
- invent new component values
|
|
189
|
+
- produce text longer than 500 characters
|
|
190
|
+
- output a degenerate single-token answer
|
|
191
|
+
|
|
192
|
+
OUTPUT a JSON array of N objects, each shaped {"raw": "...", "dropped": ["..."]}.`;
|
|
193
|
+
function buildUserPrompt(seed, n) {
|
|
194
|
+
return `INPUT:
|
|
195
|
+
${JSON.stringify({ raw: seed.raw, components: seed.components, country: seed.country }, null, 2)}
|
|
196
|
+
|
|
197
|
+
N: ${n}`;
|
|
198
|
+
}
|
|
199
|
+
function makeDeepseekProvider(model) {
|
|
200
|
+
const apiKey = $private.DEEPSEEK_API_KEY;
|
|
201
|
+
if (!apiKey)
|
|
202
|
+
throw new Error("DEEPSEEK_API_KEY env var is required for provider deepseek");
|
|
203
|
+
return {
|
|
204
|
+
name: "deepseek",
|
|
205
|
+
model,
|
|
206
|
+
async generateVariants(seed, n) {
|
|
207
|
+
const res = await fetch("https://api.deepseek.com/chat/completions", {
|
|
208
|
+
method: "POST",
|
|
209
|
+
headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
|
|
210
|
+
body: JSON.stringify({
|
|
211
|
+
model,
|
|
212
|
+
messages: [
|
|
213
|
+
{ role: "system", content: SYSTEM_PROMPT },
|
|
214
|
+
{ role: "user", content: buildUserPrompt(seed, n) },
|
|
215
|
+
],
|
|
216
|
+
response_format: { type: "json_object" },
|
|
217
|
+
max_tokens: 800,
|
|
218
|
+
temperature: 0.7,
|
|
219
|
+
}),
|
|
220
|
+
signal: AbortSignal.timeout(60_000),
|
|
221
|
+
});
|
|
222
|
+
if (!res.ok)
|
|
223
|
+
throw new Error(`DeepSeek HTTP ${res.status}: ${await res.text()}`);
|
|
224
|
+
const data = (await res.json());
|
|
225
|
+
const content = data.choices?.[0]?.message.content ?? "{}";
|
|
226
|
+
return parseCandidates(content);
|
|
227
|
+
},
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
function makeAnthropicProvider(model) {
|
|
231
|
+
const apiKey = $private.ANTHROPIC_API_KEY;
|
|
232
|
+
if (!apiKey)
|
|
233
|
+
throw new Error("ANTHROPIC_API_KEY env var is required for provider anthropic");
|
|
234
|
+
return {
|
|
235
|
+
name: "anthropic",
|
|
236
|
+
model,
|
|
237
|
+
async generateVariants(seed, n) {
|
|
238
|
+
const res = await fetch("https://api.anthropic.com/v1/messages", {
|
|
239
|
+
method: "POST",
|
|
240
|
+
headers: {
|
|
241
|
+
"x-api-key": apiKey,
|
|
242
|
+
"anthropic-version": "2023-06-01",
|
|
243
|
+
"Content-Type": "application/json",
|
|
244
|
+
},
|
|
245
|
+
body: JSON.stringify({
|
|
246
|
+
model,
|
|
247
|
+
max_tokens: 800,
|
|
248
|
+
system: SYSTEM_PROMPT,
|
|
249
|
+
messages: [{ role: "user", content: buildUserPrompt(seed, n) }],
|
|
250
|
+
}),
|
|
251
|
+
signal: AbortSignal.timeout(60_000),
|
|
252
|
+
});
|
|
253
|
+
if (!res.ok)
|
|
254
|
+
throw new Error(`Anthropic HTTP ${res.status}: ${await res.text()}`);
|
|
255
|
+
const data = (await res.json());
|
|
256
|
+
const text = data.content?.find((c) => c.type === "text")?.text ?? "[]";
|
|
257
|
+
return parseCandidates(text);
|
|
258
|
+
},
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
function parseCandidates(text) {
|
|
262
|
+
// Strip markdown fences the model sometimes wraps around JSON
|
|
263
|
+
const cleaned = text.replace(/^```(?:json)?\n?|\n?```$/g, "").trim();
|
|
264
|
+
try {
|
|
265
|
+
const parsed = JSON.parse(cleaned);
|
|
266
|
+
if (Array.isArray(parsed))
|
|
267
|
+
return parsed;
|
|
268
|
+
// Some providers wrap in {"variants": [...]} or {"candidates": [...]}
|
|
269
|
+
if (typeof parsed === "object" && parsed !== null) {
|
|
270
|
+
for (const key of ["variants", "candidates", "results"]) {
|
|
271
|
+
const v = parsed[key];
|
|
272
|
+
if (Array.isArray(v))
|
|
273
|
+
return v;
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
// fall through
|
|
279
|
+
}
|
|
280
|
+
return [];
|
|
281
|
+
}
|
|
282
|
+
// ── Validator ─────────────────────────────────────────────────────────────
|
|
283
|
+
function normalize(s) {
|
|
284
|
+
return s.toLowerCase().replace(/\s+/g, " ").trim();
|
|
285
|
+
}
|
|
286
|
+
// Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
|
|
287
|
+
const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"]);
|
|
288
|
+
function validate(seed, candidate) {
|
|
289
|
+
if (!candidate.raw || typeof candidate.raw !== "string")
|
|
290
|
+
return false;
|
|
291
|
+
if (candidate.raw.length > 500)
|
|
292
|
+
return false;
|
|
293
|
+
if (/```|<\/?\w+>|^\s*\{/.test(candidate.raw))
|
|
294
|
+
return false;
|
|
295
|
+
const normRaw = normalize(candidate.raw);
|
|
296
|
+
const dropped = new Set(candidate.dropped ?? []);
|
|
297
|
+
// LLM cannot drop required components present in the seed.
|
|
298
|
+
for (const tag of dropped) {
|
|
299
|
+
if (REQUIRED_COMPONENT_TAGS.has(tag) && seed.components[tag])
|
|
300
|
+
return false;
|
|
301
|
+
}
|
|
302
|
+
// Every kept component value must appear verbatim (post-normalization) in the candidate raw.
|
|
303
|
+
let keptCount = 0;
|
|
304
|
+
for (const [tag, value] of Object.entries(seed.components)) {
|
|
305
|
+
if (dropped.has(tag))
|
|
306
|
+
continue;
|
|
307
|
+
if (!value)
|
|
308
|
+
continue;
|
|
309
|
+
if (!normRaw.includes(normalize(value)))
|
|
310
|
+
return false;
|
|
311
|
+
keptCount++;
|
|
312
|
+
}
|
|
313
|
+
// Reject degenerate single-component candidates ("VT", "Paris" alone).
|
|
314
|
+
if (keptCount < 2)
|
|
315
|
+
return false;
|
|
316
|
+
return true;
|
|
317
|
+
}
|
|
318
|
+
// ── Main pipeline ─────────────────────────────────────────────────────────
|
|
319
|
+
export async function expandGolden(options = {}, report) {
|
|
320
|
+
const corpusPath = options.corpus ?? dataRootPath("corpus", "versioned", "v0.2.0", "corpus-v0.2.0", "test", "part-0000.parquet");
|
|
321
|
+
const count = options.count ?? 100;
|
|
322
|
+
const variants = options.variants ?? 5;
|
|
323
|
+
const providerName = options.provider ?? "deepseek";
|
|
324
|
+
const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat");
|
|
325
|
+
const concurrencyLimit = options.concurrency ?? 4;
|
|
326
|
+
const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null;
|
|
327
|
+
const ts = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
|
328
|
+
const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`;
|
|
329
|
+
const provider = providerName === "anthropic" ? makeAnthropicProvider(model) : makeDeepseekProvider(model);
|
|
330
|
+
report?.(`provider: ${provider.name} model: ${provider.model}`);
|
|
331
|
+
const seeds = await loadSeeds(corpusPath, count, includeSources, report);
|
|
332
|
+
if (seeds.length === 0) {
|
|
333
|
+
throw new Error("no seeds loaded — corpus path or filter is wrong");
|
|
334
|
+
}
|
|
335
|
+
await mkdir(dirname(outputPath), { recursive: true });
|
|
336
|
+
const outRows = [];
|
|
337
|
+
let kept = 0;
|
|
338
|
+
let dropped = 0;
|
|
339
|
+
let errored = 0;
|
|
340
|
+
// Bounded-concurrency worker pool
|
|
341
|
+
let cursor = 0;
|
|
342
|
+
const workers = Array.from({ length: Math.min(concurrencyLimit, seeds.length) }, async () => {
|
|
343
|
+
while (true) {
|
|
344
|
+
const i = cursor++;
|
|
345
|
+
if (i >= seeds.length)
|
|
346
|
+
return;
|
|
347
|
+
const seed = seeds[i];
|
|
348
|
+
try {
|
|
349
|
+
const candidates = await provider.generateVariants(seed, variants);
|
|
350
|
+
for (const cand of candidates) {
|
|
351
|
+
if (validate(seed, cand)) {
|
|
352
|
+
const goldenCandidate = {
|
|
353
|
+
raw: cand.raw,
|
|
354
|
+
components: { ...seed.components },
|
|
355
|
+
country: seed.country,
|
|
356
|
+
source: `expand-golden:${provider.name}`,
|
|
357
|
+
seed_source_id: seed.source_id,
|
|
358
|
+
seed_source_adapter: seed.source,
|
|
359
|
+
dropped_components: cand.dropped ?? [],
|
|
360
|
+
provenance: { provider: provider.name, model: provider.model },
|
|
361
|
+
};
|
|
362
|
+
// Remove dropped components from the components map
|
|
363
|
+
for (const tag of goldenCandidate.dropped_components) {
|
|
364
|
+
delete goldenCandidate.components[tag];
|
|
365
|
+
}
|
|
366
|
+
outRows.push(goldenCandidate);
|
|
367
|
+
kept++;
|
|
368
|
+
}
|
|
369
|
+
else {
|
|
370
|
+
dropped++;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
catch (err) {
|
|
375
|
+
errored++;
|
|
376
|
+
report?.(` ✗ seed ${seed.source_id}: ${err.message}`);
|
|
377
|
+
}
|
|
378
|
+
if ((i + 1) % 10 === 0) {
|
|
379
|
+
report?.(` progress: ${i + 1}/${seeds.length} kept=${kept} dropped=${dropped} errored=${errored}`);
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
await Promise.all(workers);
|
|
384
|
+
writeJSONL(outputPath, outRows);
|
|
385
|
+
report?.(`=== summary ===`);
|
|
386
|
+
report?.(`seeds processed: ${seeds.length}`);
|
|
387
|
+
report?.(`candidates kept: ${kept}`);
|
|
388
|
+
report?.(`candidates dropped (validator): ${dropped}`);
|
|
389
|
+
report?.(`seeds with errors: ${errored}`);
|
|
390
|
+
report?.(`yield: ${seeds.length > 0 ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`);
|
|
391
|
+
report?.(`output: ${outputPath}`);
|
|
392
|
+
return { seedsProcessed: seeds.length, kept, dropped, errored, outputPath };
|
|
393
|
+
}
|
|
394
|
+
//# sourceMappingURL=golden-expand.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"golden-expand.js","sourceRoot":"","sources":["../../../src/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAEnC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAA;AAiEhE,6EAA6E;AAE7E;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,MAAgB,EAAE,MAAgB;IAC3D,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,IAAI,aAAa,GAAa,EAAE,CAAA;IAChC,MAAM,KAAK,GAAG,GAAG,EAAE;QAClB,IAAI,UAAU,IAAI,aAAa,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;YACpE,GAAG,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACjD,CAAC;QACD,UAAU,GAAG,IAAI,CAAA;QACjB,aAAa,GAAG,EAAE,CAAA;IACnB,CAAC,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACxB,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YACP,SAAQ;QACT,CAAC;QACD,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YAC1C,KAAK,EAAE,CAAA;YACP,UAAU,GAAG,GAAI,CAAA;YACjB,aAAa,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IACD,KAAK,EAAE,CAAA;IAEP,OAAO,GAAG,CAAA;AACX,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,UAAkB,EAClB,KAAa,EACb,cAAkC,EAClC,MAA+B;IAE/B,MAAM,KAAK,GAAG,UAAU;SACtB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAA;IACjB,MAAM,EAAE,CAAC,sBAAsB,KAAK,CAAC,MAAM,sBAAsB,KAAK,eAAe,CAAC,CAAA;IAEtF,IAAI,cAAc,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,6BAA6B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/E,CAAC;IAED,6FAA6F;IAC7F,8FAA8F;IAC9F,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAA;IAChD,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAA;QAEjC,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAqB,CAAA;YAErD,IAAI,CAAC,GAAG;gBAAE,MAAK;YACf,OAAO,EAAE,CAAA;YAET,0FAA0F;YAC1F,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAC/D,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAA;YAEvE,4FAA4F;YAC5F,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,qBAAqB,EAAE,CAAA;gBACvB,SAAQ;YACT,CAAC;YACD,MAAM,IAAI,GAAS;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;aACxB,CAAA;YACD,IAAI,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,GAAG,EAAE,CAAA;gBACX,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;YACjC,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACF,CAAC;QACD,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,EAAE,CACP,aAAa,OAAO,gBAAgB,KAAK,CAAC,MAAM,uCAAuC,qBAAqB,EAAE,CAC9G,CAAA;IACD,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,EAAE,CAAC,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,qFAAqF;IACrF,0FAA0F;IAC1F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAClD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,KAAK,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAA;IACpD,MAAM,MAAM,GAAW,EAAE,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;QAC/B,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAElD,8EAA8E;QAC9E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAC5C;YAAA,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;QAC3C,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;QAEnC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;YACnB,MAAM,EAAE,CAAC,SAAS,GAAG,eAAe,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IACD,MAAM,EAAE,CAAC,cAAc,MAAM,CAAC,MAAM,iBAAiB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IAE9E,OAAO,MAAM,CAAA;AACd,CAAC;AAUD,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;kFAsB4D,CAAA;AAElF,SAAS,eAAe,CAAC,IAAU,EAAE,CAAS;IAC7C,OAAO;EACN,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;;KAE3F,CAAC,EAAE,CAAA;AACR,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAA;IAExC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAE1F,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,2CAA2C,EAAE;gBACpE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAClF,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK;oBACL,QAAQ,EAAE;wBACT,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;wBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE;qBACnD;oBACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;oBACxC,UAAU,EAAE,GAAG;oBACf,WAAW,EAAE,GAAG;iBAChB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACnC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAChF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0D,CAAA;YACxF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;YAE1D,OAAO,eAAe,CAAC,OAAO,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAA;IAEzC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;IAE5F,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;gBAChE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACR,WAAW,EAAE,MAAM;oBACnB,mBAAmB,EAAE,YAAY;oBACjC,cAAc,EAAE,kBAAkB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK;oBACL,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,aAAa;oBACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBAC/D,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACnC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YACjF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAwD,CAAA;YACtF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAA;YAEvE,OAAO,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7B,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAEpE,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAY,CAAA;QAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,OAAO,MAAqB,CAAA;QAEvD,sEAAsE;QACtE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACnD,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC;gBACzD,MAAM,CAAC,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAA;gBAElD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;oBAAE,OAAO,CAAgB,CAAA;YAC9C,CAAC;QACF,CAAC;IACF,CAAC;IAAC,MAAM,CAAC;QACR,eAAe;IAChB,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC;AAED,6EAA6E;AAE7E,SAAS,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACnD,CAAC;AAED,kGAAkG;AAClG,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAA;AAElG,SAAS,QAAQ,CAAC,IAAU,EAAE,SAAoB;IACjD,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,OAAO,SAAS,CAAC,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAErE,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAE5C,IAAI,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,OAAO,IAAI,EAAE,CAAC,CAAA;IAEhD,2DAA2D;IAC3D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAA;IAC3E,CAAC;IAED,6FAA6F;IAC7F,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE9B,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAA;QACrD,SAAS,EAAE,CAAA;IACZ,CAAC;IAED,uEAAuE;IACvE,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,UAA+B,EAAE,EACjC,MAA+B;IAE/B,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAA;IAC9G,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAA;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IACtC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,UAAU,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,eAAe,CAAC,CAAA;IAC7G,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACtH,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACtE,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,sCAAsC,EAAE,QAAQ,CAAA;IAErF,MAAM,QAAQ,GAAG,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,aAAa,QAAQ,CAAC,IAAI,YAAY,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,CAAC,CAAA;IAExE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,MAAM,OAAO,GAAsB,EAAE,CAAA;IACrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,kCAAkC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;YAElB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAM;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEtB,IAAI,CAAC;gBACJ,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBAElE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC/B,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;wBAC1B,MAAM,eAAe,GAAoB;4BACxC,GAAG,EAAE,IAAI,CAAC,GAAG;4BACb,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE;4BAClC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,iBAAiB,QAAQ,CAAC,IAAI,EAAE;4BACxC,cAAc,EAAE,IAAI,CAAC,SAAS;4BAC9B,mBAAmB,EAAE,IAAI,CAAC,MAAM;4BAChC,kBAAkB,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;4BACtC,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE;yBAC9D,CAAA;wBAED,oDAAoD;wBACpD,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,kBAAkB,EAAE,CAAC;4BACtD,OAAO,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;wBACvC,CAAC;wBACD,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;wBAC7B,IAAI,EAAE,CAAA;oBACP,CAAC;yBAAM,CAAC;wBACP,OAAO,EAAE,CAAA;oBACV,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACd,OAAO,EAAE,CAAA;gBACT,MAAM,EAAE,CAAC,YAAY,IAAI,CAAC,SAAS,KAAM,GAAa,CAAC,OAAO,EAAE,CAAC,CAAA;YAClE,CAAC;YAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,UAAU,IAAI,aAAa,OAAO,aAAa,OAAO,EAAE,CAAC,CAAA;YACvG,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IACF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,UAAU,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IAC/B,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,qBAAqB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7C,MAAM,EAAE,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAA;IACtD,MAAM,EAAE,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,UAAU,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAA;IACrG,MAAM,EAAE,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;AAC5E,CAAC"}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @copyright Sister Software
|
|
3
|
+
* @license AGPL-3.0
|
|
4
|
+
* @author Teffen Ellis, et al.
|
|
5
|
+
*
|
|
6
|
+
* Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
|
|
7
|
+
* human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
|
|
8
|
+
*
|
|
9
|
+
* ## What it does
|
|
10
|
+
*
|
|
11
|
+
* 1. Reads a candidates JSONL from `data/eval/golden/candidates/`
|
|
12
|
+
* 2. Reads the previous-version golden dir for forward-copy + dedup base
|
|
13
|
+
* 3. Applies filters that drop candidates unlikely to be human-typed:
|
|
14
|
+
*
|
|
15
|
+
* - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
|
|
16
|
+
* - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
|
|
17
|
+
* before locality only)
|
|
18
|
+
* - Suspicious-token signals (unmatched brackets, control chars, etc.)
|
|
19
|
+
* 4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
|
|
20
|
+
* 5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
|
|
21
|
+
* 6. Forward-copies the prior version's adversarial.jsonl + README as-is
|
|
22
|
+
* 7. Writes MANIFEST.json with sha256 of each output file
|
|
23
|
+
*
|
|
24
|
+
* ## Usage
|
|
25
|
+
*
|
|
26
|
+
* ```sh
|
|
27
|
+
* mailwoman corpus golden promote \
|
|
28
|
+
* --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
|
|
29
|
+
* --bump-to v0.1.1 \
|
|
30
|
+
* --prior v0.1.0
|
|
31
|
+
* ```
|
|
32
|
+
*/
|
|
33
|
+
export interface PromoteStats {
|
|
34
|
+
candidatesIn: number;
|
|
35
|
+
filteredOut: {
|
|
36
|
+
glued: number;
|
|
37
|
+
postcodeLeading: number;
|
|
38
|
+
suspicious: number;
|
|
39
|
+
duplicate: number;
|
|
40
|
+
forwardDup: number;
|
|
41
|
+
};
|
|
42
|
+
kept: number;
|
|
43
|
+
perCountry: Record<string, number>;
|
|
44
|
+
}
|
|
45
|
+
export interface PromoteGoldenOptions {
|
|
46
|
+
/** Candidates JSONL (required). */
|
|
47
|
+
input: string;
|
|
48
|
+
/** Target golden version dir (required, e.g. `v0.1.1`). */
|
|
49
|
+
bumpTo: string;
|
|
50
|
+
/** Previous version to forward-copy + dedup against. Default `v0.1.0`. */
|
|
51
|
+
prior?: string;
|
|
52
|
+
/** Golden dir root. Default `data/eval/golden`. */
|
|
53
|
+
goldenRoot?: string;
|
|
54
|
+
/** Skip the human-typed-likelihood filters (keep everything that passed expand-golden's validator). */
|
|
55
|
+
noFilters?: boolean;
|
|
56
|
+
/** Report what would be written but don't touch disk. */
|
|
57
|
+
dryRun?: boolean;
|
|
58
|
+
}
|
|
59
|
+
export declare function promoteGolden(options: PromoteGoldenOptions, report?: (line: string) => void): Promise<PromoteStats>;
|
|
60
|
+
//# sourceMappingURL=golden-promote.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"golden-promote.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAqBH,MAAM,WAAW,YAAY;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAClH,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAClC;AAED,MAAM,WAAW,oBAAoB;IACpC,mCAAmC;IACnC,KAAK,EAAE,MAAM,CAAA;IACb,2DAA2D;IAC3D,MAAM,EAAE,MAAM,CAAA;IACd,0EAA0E;IAC1E,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,mDAAmD;IACnD,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,uGAAuG;IACvG,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB,yDAAyD;IACzD,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAuED,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CA6JvB"}
|