@mailwoman/corpus 9.2.0 → 9.3.0

package/out/tools/golden-expand.js

@@ -0,0 +1,406 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Generate golden-set candidate entries by LLM-driven surface-form synthesis from a verified-label
+ *   seed pulled out of the corpus's labeled test split.
+ *
+ *   ## Why this approach
+ *
+ *   The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
+ *   doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
+ *   would be unverified.
+ *
+ *   This module takes the middle path:
+ *
+ *   1. **Seeds come from corpus-v0.2.0 test slice** — already through the alignment pipeline, so labels
+ *        are pipeline-verified.
+ *   2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
+ *        component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
+ *   3. **Programmatic validator drops hallucinations** — every component value must appear as a
+ *        substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
+ *        silently; cost is wasted tokens, never bad-labeled golden entries.
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   DEEPSEEK_API_KEY=sk-... \
+ *   mailwoman corpus golden expand \
+ *   --count 1000 \
+ *   --variants 5 \
+ *   --output data/eval/golden/candidates/expand-$(date +%Y%m%d-%H%M%S).jsonl
+ *   ```
+ *
+ *   ## Env
+ *
+ *   - `DEEPSEEK_API_KEY` — required for provider `deepseek`
+ *   - `ANTHROPIC_API_KEY` — required for provider `anthropic`
+ *
+ *   ## What this module does NOT do
+ *
+ *   - Does not commit anything or modify the versioned golden dir. Candidates land in
+ *       `data/eval/golden/candidates/` for operator review (skim, prune, then run
+ *       `mailwoman corpus golden promote`).
+ *   - Does not score the LLM's quality — that's an eyeball job after pilot lands.
+ *   - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
+ */
+import { APIClient, pluckResponseData } from "@mailwoman/core/api";
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { makeDirectories } from "@mailwoman/core/fs/writers";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { isPresent } from "@mailwoman/core/objects";
+import { foldCaseWhitespace } from "@mailwoman/normalize/fold";
+import { dirname } from "path-ts";
+import { createNewlineWriter } from "spliterator";
+import { $private } from "#env";
+import { streamParquetRows } from "#utils/parquet";
+// ── Types ─────────────────────────────────────────────────────────────────
+/**
+ * Longest raw line still plausibly a single address rather than a concatenated record.
+ */
+const MAX_CANDIDATE_LENGTH = 500;
+// ── Seed loading ──────────────────────────────────────────────────────────
+/**
+ * Decode BIO labels + tokens into a verified components map. Mirrors the Python `decode_components` in
+ * mailwoman_train/eval.py — first-occurrence-wins per tag, contiguous B-X/I-X runs concatenated with a single space
+ * (the canonical separator used by corpus alignment).
+ */
+function decodeComponents(tokens, labels) {
+    const out = {};
+    let currentTag = null;
+    let currentTokens = [];
+    const flush = () => {
+        if (currentTag && currentTokens.length && !(currentTag in out)) {
+            out[currentTag] = currentTokens.join(" ").trim();
+        }
+        currentTag = null;
+        currentTokens = [];
+    };
+    for (let i = 0; i < labels.length; i++) {
+        const label = labels[i];
+        const tok = tokens[i] ?? "";
+        if (label === "O") {
+            flush();
+            continue;
+        }
+        const [prefix, tag] = label.split("-", 2);
+        if (prefix === "B" || currentTag !== tag) {
+            flush();
+            currentTag = tag;
+            currentTokens = [tok];
+        }
+        else {
+            currentTokens.push(tok);
+        }
+    }
+    flush();
+    return out;
+}
+async function loadSeeds(corpusPath, count, includeSources, report) {
+    const paths = corpusPath
+        .split(",")
+        .map((p) => p.trim())
+        .filter(isPresent);
+    report?.(`reading seeds from ${paths.length} slice(s) (target: ${count}, stratified)`);
+    if (includeSources) {
+        report?.(`  include-sources filter: ${Array.from(includeSources).join(", ")}`);
+    }
+    // Stratified sampling: read all rows from all slices, group by source. Bounded by per-source
+    // reservoir: keep at most max(2*count, 5000) rows per source so we don't blow memory on train
+    // slices (1M rows × many slices). Sampling later is uniform within each pool.
+    const bySource = new Map();
+    const PER_SOURCE_CAP = Math.max(2 * count, 5000);
+    let scanned = 0;
+    let skippedThinComponents = 0;
+    for (const path of paths) {
+        for await (const row of streamParquetRows(path)) {
+            scanned++;
+            // Source allow-list (--include-sources) — applied early to skip parsing rows we won't use
+            if (includeSources && !includeSources.has(row.source))
+                continue;
+            const components = decodeComponents(row.tokens ?? [], row.labels ?? []);
+            // Skip rows with too few components — single-name wof-admin entries don't make useful seeds
+            if (Object.keys(components).length < 2) {
+                skippedThinComponents++;
+                continue;
+            }
+            const seed = {
+                raw: row.raw,
+                components,
+                country: row.country,
+                source: row.source,
+                source_id: row.source_id,
+            };
+            let bucket = bySource.get(row.source);
+            if (!bucket) {
+                bucket = [];
+                bySource.set(row.source, bucket);
+            }
+            if (bucket.length < PER_SOURCE_CAP) {
+                bucket.push(seed);
+            }
+        }
+    }
+    report?.(`  scanned ${scanned} rows across ${paths.length} slice(s); thin-components dropped: ${skippedThinComponents}`);
+    report?.(`  per-source pool sizes:`);
+    for (const [src, pool] of bySource) {
+        report?.(`    ${src}: ${pool.length}`);
+    }
+    // Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
+    // to sources in alphabetical order. If a pool is smaller than its target, take all of it.
+    const sources = Array.from(bySource.keys()).toSorted();
+    const perSource = Math.floor(count / sources.length);
+    const remainder = count - perSource * sources.length;
+    const picked = [];
+    for (let i = 0; i < sources.length; i++) {
+        const src = sources[i];
+        const pool = bySource.get(src);
+        const target = perSource + (i < remainder ? 1 : 0);
+        // Random subsample without replacement — deterministic via shuffle then slice
+        for (let j = pool.length - 1; j > 0; j--) {
+            const k = Math.floor(Math.random() * (j + 1));
+            [pool[j], pool[k]] = [pool[k], pool[j]];
+        }
+        const take = Math.min(target, pool.length);
+        picked.push(...pool.slice(0, take));
+        if (take < target) {
+            report?.(`    ⚠ ${src}: requested ${target}, pool had ${pool.length}`);
+        }
+    }
+    report?.(`  → loaded ${picked.length} seeds across ${sources.length} sources`);
+    return picked;
+}
+const SYSTEM_PROMPT = `You are a postal-address surface-form generator. Given a structured address, produce realistic variants a human might type into a geocoder.
+
+CONSTRAINT — you MUST preserve every kept component value verbatim in the output. You may:
+- vary case (UPPER, lower, Title Case)
+- abbreviate (Saint → St, Avenue → Ave, Boulevard → Blvd, North → N, etc.)
+- vary punctuation (commas, dashes, spaces, line breaks)
+- reorder components (postcode-first, country-first, address-only)
+- drop OPTIONAL components — list them in "dropped"
+
+OPTIONAL components (allowed to drop): country, postcode, dependent_locality, subregion, cedex.
+REQUIRED components (must keep, even if input has them): locality, region (when present),
+street, house_number, venue (when present).
+
+ALWAYS keep AT LEAST 2 components in the final raw text. Single-component variants like
+"VT" or "Paris" are USELESS as eval entries — do not produce them.
+
+You MUST NOT:
+- introduce typos (validator drops these silently — wasted tokens)
+- invent new component values
+- produce text longer than 500 characters
+- output a degenerate single-token answer
+
+OUTPUT a JSON array of N objects, each shaped {"raw": "...", "dropped": ["..."]}.`;
+function buildUserPrompt(seed, n) {
+    return `INPUT:
+${JSON.stringify({ raw: seed.raw, components: seed.components, country: seed.country }, null, 2)}
+
+N: ${n}`;
+}
+function makeDeepseekProvider(model) {
+    const apiKey = $private.DEEPSEEK_API_KEY;
+    if (!apiKey)
+        throw new Error("DEEPSEEK_API_KEY env var is required for provider deepseek");
+    return {
+        name: "deepseek",
+        model,
+        async generateVariants(seed, n) {
+            const data = await new APIClient({
+                displayName: "deepseek",
+                retry: true,
+                axios: { headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" } },
+            })
+                .fetch({
+                url: "https://api.deepseek.com/chat/completions",
+                method: "post",
+                timeout: 60_000,
+                data: {
+                    model,
+                    messages: [
+                        { role: "system", content: SYSTEM_PROMPT },
+                        { role: "user", content: buildUserPrompt(seed, n) },
+                    ],
+                    response_format: { type: "json_object" },
+                    max_tokens: 800,
+                    temperature: 0.7,
+                },
+            })
+                .then(pluckResponseData);
+            const content = data.choices?.[0]?.message.content ?? "{}";
+            return parseCandidates(content);
+        },
+    };
+}
+function makeAnthropicProvider(model) {
+    const apiKey = $private.ANTHROPIC_API_KEY;
+    if (!apiKey)
+        throw new Error("ANTHROPIC_API_KEY env var is required for provider anthropic");
+    return {
+        name: "anthropic",
+        model,
+        async generateVariants(seed, n) {
+            const data = await new APIClient({
+                displayName: "anthropic",
+                retry: true,
+                axios: {
+                    headers: {
+                        "x-api-key": apiKey,
+                        "anthropic-version": "2023-06-01",
+                        "Content-Type": "application/json",
+                    },
+                },
+            })
+                .fetch({
+                url: "https://api.anthropic.com/v1/messages",
+                method: "post",
+                timeout: 60_000,
+                data: {
+                    model,
+                    max_tokens: 800,
+                    system: SYSTEM_PROMPT,
+                    messages: [{ role: "user", content: buildUserPrompt(seed, n) }],
+                },
+            })
+                .then(pluckResponseData);
+            const text = data.content?.find((c) => c.type === "text")?.text ?? "[]";
+            return parseCandidates(text);
+        },
+    };
+}
+function parseCandidates(text) {
+    // Strip markdown fences the model sometimes wraps around JSON
+    const cleaned = text.replaceAll(/^```(?:json)?\n?|\n?```$/g, "").trim();
+    const parsed = tryParsingJSON(cleaned);
+    if (Array.isArray(parsed))
+        return parsed;
+    // Some providers wrap in {"variants": [...]} or {"candidates": [...]}
+    if (typeof parsed === "object" && parsed !== null) {
+        for (const key of ["variants", "candidates", "results"]) {
+            const v = parsed[key];
+            if (Array.isArray(v))
+                return v;
+        }
+    }
+    return [];
+}
+// ── Validator ─────────────────────────────────────────────────────────────
+/**
+ * Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
+ */
+const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"]);
+function validate(seed, candidate) {
+    if (!candidate.raw || typeof candidate.raw !== "string")
+        return false;
+    if (candidate.raw.length > MAX_CANDIDATE_LENGTH)
+        return false;
+    if (/```|<\/?\w+>|^\s*\{/.test(candidate.raw))
+        return false;
+    const normRaw = foldCaseWhitespace(candidate.raw);
+    const dropped = new Set(candidate.dropped);
+    // LLM cannot drop required components present in the seed.
+    for (const tag of dropped) {
+        if (REQUIRED_COMPONENT_TAGS.has(tag) && seed.components[tag])
+            return false;
+    }
+    // Every kept component value must appear verbatim (post-normalization) in the candidate raw.
+    let keptCount = 0;
+    for (const [tag, value] of Object.entries(seed.components)) {
+        if (dropped.has(tag))
+            continue;
+        if (!value)
+            continue;
+        if (!normRaw.includes(foldCaseWhitespace(value)))
+            return false;
+        keptCount++;
+    }
+    // Reject degenerate single-component candidates ("VT", "Paris" alone).
+    if (keptCount < 2)
+        return false;
+    return true;
+}
+// ── Main pipeline ─────────────────────────────────────────────────────────
+export async function expandGolden(options = {}, report) {
+    const corpusPath = (options.corpus ?? dataRootPath("corpus", "versioned", "v0.2.0", "corpus-v0.2.0", "test", "part-0000.parquet")).toString();
+    const count = options.count ?? 100;
+    const variants = options.variants ?? 5;
+    const providerName = options.provider ?? "deepseek";
+    const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat");
+    const concurrencyLimit = options.concurrency ?? 4;
+    const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null;
+    const ts = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19);
+    const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`;
+    const provider = providerName === "anthropic" ? makeAnthropicProvider(model) : makeDeepseekProvider(model);
+    report?.(`provider: ${provider.name}  model: ${provider.model}`);
+    const seeds = await loadSeeds(corpusPath, count, includeSources, report);
+    if (!seeds.length) {
+        throw new Error("no seeds loaded — corpus path or filter is wrong");
+    }
+    await makeDirectories(dirname(outputPath));
+    const outRows = [];
+    let kept = 0;
+    let dropped = 0;
+    let errored = 0;
+    // Bounded-concurrency worker pool
+    let cursor = 0;
+    const workers = Array.from({ length: Math.min(concurrencyLimit, seeds.length) }, async () => {
+        while (true) {
+            const i = cursor++;
+            if (i >= seeds.length)
+                return;
+            const seed = seeds[i];
+            try {
+                const candidates = await provider.generateVariants(seed, variants);
+                for (const cand of candidates) {
+                    if (validate(seed, cand)) {
+                        const goldenCandidate = {
+                            raw: cand.raw,
+                            components: { ...seed.components },
+                            country: seed.country,
+                            source: `expand-golden:${provider.name}`,
+                            seed_source_id: seed.source_id,
+                            seed_source_adapter: seed.source,
+                            dropped_components: cand.dropped ?? [],
+                            provenance: { provider: provider.name, model: provider.model },
+                        };
+                        // Remove dropped components from the components map
+                        for (const tag of goldenCandidate.dropped_components) {
+                            // oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record; the object is not on a hot path
+                            delete goldenCandidate.components[tag];
+                        }
+                        outRows.push(goldenCandidate);
+                        kept++;
+                    }
+                    else {
+                        dropped++;
+                    }
+                }
+            }
+            catch (error) {
+                errored++;
+                report?.(`  ✗ seed ${seed.source_id}: ${error.message}`);
+            }
+            if ((i + 1) % 10 === 0) {
+                report?.(`  progress: ${i + 1}/${seeds.length}  kept=${kept}  dropped=${dropped}  errored=${errored}`);
+            }
+        }
+    });
+    await Promise.all(workers);
+    {
+        await using out = createNewlineWriter(outputPath);
+        for (const row of outRows) {
+            await out.write(JSON.stringify(row));
+        }
+    }
+    report?.(`=== summary ===`);
+    report?.(`seeds processed:  ${seeds.length}`);
+    report?.(`candidates kept:  ${kept}`);
+    report?.(`candidates dropped (validator): ${dropped}`);
+    report?.(`seeds with errors: ${errored}`);
+    report?.(`yield: ${seeds.length ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`);
+    report?.(`output:           ${outputPath}`);
+    return { seedsProcessed: seeds.length, kept, dropped, errored, outputPath };
+}
+//# sourceMappingURL=golden-expand.js.map

package/out/tools/golden-expand.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-expand.js","sourceRoot":"","sources":["../../lib/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,OAAO,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAA;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAA;AACnD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEjD,OAAO,EAAE,QAAQ,EAAE,MAAM,MAAM,CAAA;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAElD,6EAA6E;AAE7E;;GAEG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAA;AA+EhC,6EAA6E;AAE7E;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,MAAgB,EAAE,MAAgB;IAC3D,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,IAAI,aAAa,GAAa,EAAE,CAAA;IAEhC,MAAM,KAAK,GAAG,GAAG,EAAE;QAClB,IAAI,UAAU,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACjD,CAAC;QAED,UAAU,GAAG,IAAI,CAAA;QACjB,aAAa,GAAG,EAAE,CAAA;IACnB,CAAC,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACxB,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YAEP,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YAC1C,KAAK,EAAE,CAAA;YACP,UAAU,GAAG,GAAI,CAAA;YACjB,aAAa,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,KAAK,EAAE,CAAA;IAEP,OAAO,GAAG,CAAA;AACX,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,UAAkB,EAClB,KAAa,EACb,cAAkC,EAClC,MAA+B;IAE/B,MAAM,KAAK,GAAG,UAAU;SACtB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,SAAS,CAAC,CAAA;IAEnB,MAAM,EAAE,CAAC,sBAAsB,KAAK,CAAC,MAAM,sBAAsB,KAAK,eAAe,CAAC,CAAA;IAEtF,IAAI,cAAc,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,6BAA6B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/E,CAAC;IAED,6FAA6F;IAC7F,8FAA8F;IAC9F,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAA;IAChD,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,iBAAiB,CAAY,IAAI,CAAC,EAAE,CAAC;YAC5D,OAAO,EAAE,CAAA;YAET,0FAA0F;YAC1F,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAC/D,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAA;YAEvE,4FAA4F;YAC5F,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,qBAAqB,EAAE,CAAA;gBAEvB,SAAQ;YACT,CAAC;YAED,MAAM,IAAI,GAAS;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;aACxB,CAAA;YAED,IAAI,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,GAAG,EAAE,CAAA;gBACX,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;YACjC,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACF,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CACP,aAAa,OAAO,gBAAgB,KAAK,CAAC,MAAM,uCAAuC,qBAAqB,EAAE,CAC9G,CAAA;IAED,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,EAAE,CAAC,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,qFAAqF;IACrF,0FAA0F;IAC1F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,KAAK,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAA;IACpD,MAAM,MAAM,GAAW,EAAE,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;QAC/B,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAElD,8EAA8E;QAC9E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAC5C;YAAA,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;QAC3C,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;QAEnC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;YACnB,MAAM,EAAE,CAAC,SAAS,GAAG,eAAe,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,cAAc,MAAM,CAAC,MAAM,iBAAiB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IAE9E,OAAO,MAAM,CAAA;AACd,CAAC;AAUD,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;kFAsB4D,CAAA;AAElF,SAAS,eAAe,CAAC,IAAU,EAAE,CAAS;IAC7C,OAAO;EACN,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;;KAE3F,CAAC,EAAE,CAAA;AACR,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAA;IAExC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAE1F,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC;gBAChC,WAAW,EAAE,UAAU;gBACvB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE,EAAE,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,EAAE;aAC7F,CAAC;iBACA,KAAK,CAAwD;gBAC7D,GAAG,EAAE,2CAA2C;gBAChD,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE;oBACL,KAAK;oBACL,QAAQ,EAAE;wBACT,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;wBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE;qBACnD;oBACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;oBACxC,UAAU,EAAE,GAAG;oBACf,WAAW,EAAE,GAAG;iBAChB;aACD,CAAC;iBACD,IAAI,CAAC,iBAAiB,CAAC,CAAA;YAEzB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;YAE1D,OAAO,eAAe,CAAC,OAAO,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAA;IAEzC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;IAE5F,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,IAAI,SAAS,CAAC;gBAChC,WAAW,EAAE,WAAW;gBACxB,KAAK,EAAE,IAAI;gBACX,KAAK,EAAE;oBACN,OAAO,EAAE;wBACR,WAAW,EAAE,MAAM;wBACnB,mBAAmB,EAAE,YAAY;wBACjC,cAAc,EAAE,kBAAkB;qBAClC;iBACD;aACD,CAAC;iBACA,KAAK,CAAsD;gBAC3D,GAAG,EAAE,uCAAuC;gBAC5C,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,MAAM;gBACf,IAAI,EAAE;oBACL,KAAK;oBACL,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,aAAa;oBACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBAC/D;aACD,CAAC;iBACD,IAAI,CAAC,iBAAiB,CAAC,CAAA;YAEzB,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAA;YAEvE,OAAO,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7B,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAEvE,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;IAEtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAqB,CAAA;IAEvD,sEAAsE;IACtE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAA;YAElD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAgB,CAAA;QAC9C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC;AAED,6EAA6E;AAE7E;;GAEG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAA;AAElG,SAAS,QAAQ,CAAC,IAAU,EAAE,SAAoB;IACjD,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,OAAO,SAAS,CAAC,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAErE,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,oBAAoB;QAAE,OAAO,KAAK,CAAA;IAE7D,IAAI,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAC3D,MAAM,OAAO,GAAG,kBAAkB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACjD,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE1C,2DAA2D;IAC3D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAA;IAC3E,CAAC;IAED,6FAA6F;IAC7F,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE9B,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAA;QAE9D,SAAS,EAAE,CAAA;IACZ,CAAC;IAED,uEAAuE;IACvE,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,UAA+B,EAAE,EACjC,MAA+B;IAE/B,MAAM,UAAU,GAAG,CAClB,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAC7G,CAAC,QAAQ,EAAE,CAAA;IAEZ,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAA;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IACtC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,UAAU,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,eAAe,CAAC,CAAA;IAC7G,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACtH,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACzE,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,sCAAsC,EAAE,QAAQ,CAAA;IAErF,MAAM,QAAQ,GAAG,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,aAAa,QAAQ,CAAC,IAAI,YAAY,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,CAAC,CAAA;IAExE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,eAAe,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,CAAA;IAC1C,MAAM,OAAO,GAAsB,EAAE,CAAA;IACrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,kCAAkC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;YAElB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAM;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEtB,IAAI,CAAC;gBACJ,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBAElE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC/B,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;wBAC1B,MAAM,eAAe,GAAoB;4BACxC,GAAG,EAAE,IAAI,CAAC,GAAG;4BACb,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE;4BAClC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,iBAAiB,QAAQ,CAAC,IAAI,EAAE;4BACxC,cAAc,EAAE,IAAI,CAAC,SAAS;4BAC9B,mBAAmB,EAAE,IAAI,CAAC,MAAM;4BAChC,kBAAkB,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;4BACtC,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE;yBAC9D,CAAA;wBAED,oDAAoD;wBACpD,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,kBAAkB,EAAE,CAAC;4BACtD,iIAAiI;4BACjI,OAAO,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;wBACvC,CAAC;wBAED,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;wBAE7B,IAAI,EAAE,CAAA;oBACP,CAAC;yBAAM,CAAC;wBACP,OAAO,EAAE,CAAA;oBACV,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;gBACT,MAAM,EAAE,CAAC,YAAY,IAAI,CAAC,SAAS,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YACpE,CAAC;YAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,UAAU,IAAI,aAAa,OAAO,aAAa,OAAO,EAAE,CAAC,CAAA;YACvG,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,CAAC;QACA,YAAY,GAAG,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAA;QAEjD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,qBAAqB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7C,MAAM,EAAE,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAA;IACtD,MAAM,EAAE,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,UAAU,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAA;IACjG,MAAM,EAAE,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;AAC5E,CAAC"}

package/out/tools/golden-promote.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"golden-promote.d.ts","sourceRoot":"","sources":["../../lib/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AA4BH,MAAM,WAAW,YAAY;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAClH,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAClC;AAED,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AA0DD,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAwLvB"}

package/out/tools/golden-promote.js

@@ -0,0 +1,240 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
+ *   human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
+ *
+ *   ## What it does
+ *
+ *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
+ *   2. Reads the previous-version golden dir for forward-copy + dedup base
+ *   3. Applies filters that drop candidates unlikely to be human-typed:
+ *
+ *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
+ *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
+ *                 before locality only)
+ *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
+ *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
+ *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
+ *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
+ *   7. Writes MANIFEST.json with sha256 of each output file
+ *
+ *   ## Usage
+ *
+ *   ```sh
+ *   mailwoman corpus golden promote \
+ *   --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
+ *   --bump-to v0.1.1 \
+ *   --prior v0.1.0
+ *   ```
+ */
+import { readDirectory, pathExists } from "@mailwoman/core/fs/readers";
+import { copyFileTo, makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { foldCaseWhitespace } from "@mailwoman/normalize/fold";
+import { join } from "path-ts";
+import { createNewlineWriter, JSONSpliterator } from "spliterator";
+// ── Types ──────────────────────────────────────────────────────────────────
+/**
+ * Components a candidate needs before it is rich enough to be worth a golden-set slot.
+ */
+const MIN_GOLDEN_COMPONENTS = 5;
+/**
+ * Components an existing entry must still carry to survive promotion.
+ */
+const MIN_PROMOTABLE_COMPONENTS = 4;
+/**
+ * Quote-delimited segments above which the raw line is malformed rather than merely quoted.
+ */
+const MAX_QUOTE_SEGMENTS = 3;
+// ── Filters ───────────────────────────────────────────────────────────────
+/**
+ * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
+ * together rather than human-typed.
+ */
+function isComponentsGlued(entry) {
+    const componentCount = Object.keys(entry.components).length;
+    if (componentCount < MIN_GOLDEN_COMPONENTS)
+        return false;
+    const separators = (entry.raw.match(/[,\n;]/g) ?? []).length;
+    return separators < 2;
+}
+/**
+ * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
+ * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
+ * often precedes locality there).
+ */
+function isPostcodeBadlyLeading(entry) {
+    if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS)
+        return false;
+    if (entry.country === "FR" || entry.country === "France")
+        return false;
+    const postcode = entry.components.postcode;
+    if (!postcode)
+        return false;
+    const postcodeIdx = entry.raw.indexOf(postcode);
+    if (postcodeIdx === -1)
+        return false;
+    return postcodeIdx < entry.raw.length / 3;
+}
+/**
+ * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
+ * suggests the LLM emitted markup instead of an address.
+ */
+function isSuspicious(entry) {
+    const raw = entry.raw;
+    // eslint-disable-next-line no-control-regex
+    if (/[\u0000-\u001F\u007F]/.test(raw))
+        return true;
+    const openBrackets = (raw.match(/[[({<]/g) ?? []).length;
+    const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length;
+    if (openBrackets !== closeBrackets)
+        return true;
+    if (raw.split('"').length > MAX_QUOTE_SEGMENTS)
+        return true;
+    // too many quote marks
+    return false;
+}
+// ── Main ───────────────────────────────────────────────────────────────────
+export async function promoteGolden(options, report) {
+    const prior = options.prior ?? "v0.1.0";
+    const goldenRoot = options.goldenRoot ?? "data/eval/golden";
+    const applyFilters = !options.noFilters;
+    const dryRun = options.dryRun ?? false;
+    report?.(`reading candidates: ${options.input}`);
+    // The candidates file is the REQUIRED input, so its absence must not read as "zero candidates" —
+    // that promotes the prior version unchanged and reports success. `JSONSpliterator.fromAsync`
+    // reports a missing path as a bare `TypeError` naming its own internals, so name the path here.
+    if (!(await pathExists(options.input))) {
+        throw new Error(`Candidates file not found: ${options.input}`);
+    }
+    const candidates = await Array.fromAsync(JSONSpliterator.fromAsync(options.input));
+    report?.(`  ${candidates.length} candidates loaded`);
+    // Forward-copy base: existing entries from the prior golden version go forward verbatim,
+    // and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
+    const priorDir = join(goldenRoot, prior);
+    const priorEntries = [];
+    const seenNormalized = new Set();
+    if (await pathExists(priorDir)) {
+        for (const f of (await readDirectory(priorDir)).filter((n) => n.endsWith(".jsonl"))) {
+            const country = f.replace(".jsonl", "").toUpperCase();
+            const entries = await Array.fromAsync(JSONSpliterator.fromAsync(join(priorDir, f)));
+            priorEntries.push({ country, entries });
+            for (const e of entries) {
+                seenNormalized.add(foldCaseWhitespace(e.raw));
+            }
+            report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`);
+        }
+    }
+    else {
+        report?.(`  ⚠ prior dir ${priorDir} not found — starting fresh`);
+    }
+    // Filter pass
+    const stats = {
+        candidatesIn: candidates.length,
+        filteredOut: { glued: 0, postcodeLeading: 0, suspicious: 0, duplicate: 0, forwardDup: 0 },
+        kept: 0,
+        perCountry: {},
+    };
+    const accepted = [];
+    const seenInBatch = new Set();
+    for (const cand of candidates) {
+        const norm = foldCaseWhitespace(cand.raw);
+        // Dedup pass 1: against prior versioned golden
+        if (seenNormalized.has(norm)) {
+            stats.filteredOut.forwardDup++;
+            continue;
+        }
+        // Dedup pass 2: against this batch
+        if (seenInBatch.has(norm)) {
+            stats.filteredOut.duplicate++;
+            continue;
+        }
+        if (applyFilters) {
+            if (isComponentsGlued(cand)) {
+                stats.filteredOut.glued++;
+                continue;
+            }
+            if (isPostcodeBadlyLeading(cand)) {
+                stats.filteredOut.postcodeLeading++;
+                continue;
+            }
+            if (isSuspicious(cand)) {
+                stats.filteredOut.suspicious++;
+                continue;
+            }
+        }
+        accepted.push(cand);
+        seenInBatch.add(norm);
+        stats.kept++;
+        const country = cand.country || "OTHER";
+        stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1;
+    }
+    // Bucket per country, including forward-copied entries
+    const buckets = new Map();
+    for (const { country, entries } of priorEntries) {
+        // Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
+        buckets.set(country, [...entries]);
+    }
+    for (const cand of accepted) {
+        const key = (cand.country || "OTHER").toUpperCase();
+        if (!buckets.has(key)) {
+            buckets.set(key, []);
+        }
+        buckets.get(key).push(cand);
+    }
+    // Output
+    const outDir = join(goldenRoot, options.bumpTo);
+    report?.(`=== plan ===`);
+    report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`);
+    for (const [key, entries] of buckets) {
+        report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`);
+    }
+    report?.(`=== filter stats ===`);
+    report?.(`candidates in:        ${stats.candidatesIn}`);
+    report?.(`  filtered (glued):           ${stats.filteredOut.glued}`);
+    report?.(`  filtered (postcode-lead):   ${stats.filteredOut.postcodeLeading}`);
+    report?.(`  filtered (suspicious):      ${stats.filteredOut.suspicious}`);
+    report?.(`  filtered (dup-in-batch):    ${stats.filteredOut.duplicate}`);
+    report?.(`  filtered (dup-vs-prior):    ${stats.filteredOut.forwardDup}`);
+    report?.(`  kept:                       ${stats.kept}`);
+    report?.(`per-country (kept):`);
+    for (const [c, n] of Object.entries(stats.perCountry).toSorted((a, b) => b[1] - a[1])) {
+        report?.(`  ${c}: ${n}`);
+    }
+    if (dryRun) {
+        report?.(`(dry-run: no files written)`);
+        return stats;
+    }
+    await makeDirectories(outDir);
+    const manifest = {
+        promoted_at: new Date().toISOString(),
+        from: options.input,
+        from_sha256: await sha256File(options.input),
+        files: {},
+    };
+    for (const [key, entries] of buckets) {
+        const filename = `${key.toLowerCase()}.jsonl`;
+        const path = join(outDir, filename);
+        {
+            await using out = createNewlineWriter(path);
+            for (const entry of entries) {
+                await out.write(JSON.stringify(entry));
+            }
+        }
+        manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) };
+    }
+    // Forward-copy non-.jsonl files (README.md, etc.) from prior
+    if (await pathExists(priorDir)) {
+        for (const f of (await readDirectory(priorDir)).filter((n) => !n.endsWith(".jsonl"))) {
+            await copyFileTo(join(priorDir, f), join(outDir, f));
+            report?.(`  forward-copied: ${f}`);
+        }
+    }
+    await writeLocalJSONFile(manifest, join(outDir, "MANIFEST.json"));
+    report?.(`✓ promoted to ${outDir}`);
+    return stats;
+}
+//# sourceMappingURL=golden-promote.js.map