@mailwoman/corpus 9.1.0 → 9.3.0

package/out/src/tools/golden-expand.js

@@ -1,404 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Generate golden-set candidate entries by LLM-driven surface-form synthesis from a verified-label
- *   seed pulled out of the corpus's labeled test split.
- *
- *   ## Why this approach
- *
- *   The Phase 2 golden set has 74 entries; session-notes.md called for ≥500/locale. Manual curation
- *   doesn't scale. Pure-LLM generation (invent raw + labels from scratch) is too noisy — labels
- *   would be unverified.
- *
- *   This module takes the middle path:
- *
- *   1. **Seeds come from corpus-v0.2.0 test shard** — already through the alignment pipeline, so labels
- *        are pipeline-verified.
- *   2. **LLM only varies the surface form** — case, abbreviations, reordering, dropped components. The
- *        component VALUES (locality string, postcode digits, etc.) are preserved verbatim.
- *   3. **Programmatic validator drops hallucinations** — every component value must appear as a
- *        substring (case-insensitive, whitespace-normalized) of the variant's raw. Failures dropped
- *        silently; cost is wasted tokens, never bad-labeled golden entries.
- *
- *   ## Usage
- *
- *   ```sh
- *   DEEPSEEK_API_KEY=sk-... \
- *   mailwoman corpus golden expand \
- *   --count 1000 \
- *   --variants 5 \
- *   --output data/eval/golden/candidates/expand-$(date +%Y%m%d-%H%M%S).jsonl
- *   ```
- *
- *   ## Env
- *
- *   - `DEEPSEEK_API_KEY` — required for provider `deepseek`
- *   - `ANTHROPIC_API_KEY` — required for provider `anthropic`
- *
- *   ## What this module does NOT do
- *
- *   - Does not commit anything or modify the versioned golden dir. Candidates land in
- *       `data/eval/golden/candidates/` for operator review (skim, prune, then run
- *       `mailwoman corpus golden promote`).
- *   - Does not score the LLM's quality — that's an eyeball job after pilot lands.
- *   - Does not retry hallucinated candidates. Cost of wasted tokens is trivial (~$0.0006/each).
- */
-import { mkdir } from "node:fs/promises";
-import { dirname } from "node:path";
-import { ParquetReader } from "@dsnp/parquetjs";
-import { $private } from "@mailwoman/core/env";
-import { tryParsingJSON } from "@mailwoman/core/objects";
-import { dataRootPath } from "@mailwoman/core/utils";
-import { createNewlineWriter } from "spliterator";
-// ── Types ─────────────────────────────────────────────────────────────────
-/**
- * Longest raw line still plausibly a single address rather than a concatenated record.
- */
-const MAX_CANDIDATE_LENGTH = 500;
-// ── Seed loading ──────────────────────────────────────────────────────────
-/**
- * Decode BIO labels + tokens into a verified components map. Mirrors the Python `decode_components` in
- * mailwoman_train/eval.py — first-occurrence-wins per tag, contiguous B-X/I-X runs concatenated with a single space
- * (the canonical separator used by corpus alignment).
- */
-function decodeComponents(tokens, labels) {
-    const out = {};
-    let currentTag = null;
-    let currentTokens = [];
-    const flush = () => {
-        if (currentTag && currentTokens.length && !(currentTag in out)) {
-            out[currentTag] = currentTokens.join(" ").trim();
-        }
-        currentTag = null;
-        currentTokens = [];
-    };
-    for (let i = 0; i < labels.length; i++) {
-        const label = labels[i];
-        const tok = tokens[i] ?? "";
-        if (label === "O") {
-            flush();
-            continue;
-        }
-        const [prefix, tag] = label.split("-", 2);
-        if (prefix === "B" || currentTag !== tag) {
-            flush();
-            currentTag = tag;
-            currentTokens = [tok];
-        }
-        else {
-            currentTokens.push(tok);
-        }
-    }
-    flush();
-    return out;
-}
-async function loadSeeds(corpusPath, count, includeSources, report) {
-    const paths = corpusPath
-        .split(",")
-        .map((p) => p.trim())
-        .filter(Boolean);
-    report?.(`reading seeds from ${paths.length} shard(s) (target: ${count}, stratified)`);
-    if (includeSources) {
-        report?.(`  include-sources filter: ${Array.from(includeSources).join(", ")}`);
-    }
-    // Stratified sampling: read all rows from all shards, group by source. Bounded by per-source
-    // reservoir: keep at most max(2*count, 5000) rows per source so we don't blow memory on train
-    // shards (1M rows × many shards). Sampling later is uniform within each pool.
-    const bySource = new Map();
-    const PER_SOURCE_CAP = Math.max(2 * count, 5000);
-    let scanned = 0;
-    let skippedThinComponents = 0;
-    for (const path of paths) {
-        const reader = await ParquetReader.openFile(path);
-        const cursor = reader.getCursor();
-        while (true) {
-            const row = (await cursor.next());
-            if (!row)
-                break;
-            scanned++;
-            // Source allow-list (--include-sources) — applied early to skip parsing rows we won't use
-            if (includeSources && !includeSources.has(row.source))
-                continue;
-            const components = decodeComponents(row.tokens ?? [], row.labels ?? []);
-            // Skip rows with too few components — single-name wof-admin entries don't make useful seeds
-            if (Object.keys(components).length < 2) {
-                skippedThinComponents++;
-                continue;
-            }
-            const seed = {
-                raw: row.raw,
-                components,
-                country: row.country,
-                source: row.source,
-                source_id: row.source_id,
-            };
-            let bucket = bySource.get(row.source);
-            if (!bucket) {
-                bucket = [];
-                bySource.set(row.source, bucket);
-            }
-            if (bucket.length < PER_SOURCE_CAP) {
-                bucket.push(seed);
-            }
-        }
-        await reader.close();
-    }
-    report?.(`  scanned ${scanned} rows across ${paths.length} shard(s); thin-components dropped: ${skippedThinComponents}`);
-    report?.(`  per-source pool sizes:`);
-    for (const [src, pool] of bySource) {
-        report?.(`    ${src}: ${pool.length}`);
-    }
-    // Round-robin sample. Each source gives floor(count / nSources) seeds; rounding goes
-    // to sources in alphabetical order. If a pool is smaller than its target, take all of it.
-    const sources = Array.from(bySource.keys()).toSorted();
-    const perSource = Math.floor(count / sources.length);
-    const remainder = count - perSource * sources.length;
-    const picked = [];
-    for (let i = 0; i < sources.length; i++) {
-        const src = sources[i];
-        const pool = bySource.get(src);
-        const target = perSource + (i < remainder ? 1 : 0);
-        // Random subsample without replacement — deterministic via shuffle then slice
-        for (let j = pool.length - 1; j > 0; j--) {
-            const k = Math.floor(Math.random() * (j + 1));
-            [pool[j], pool[k]] = [pool[k], pool[j]];
-        }
-        const take = Math.min(target, pool.length);
-        picked.push(...pool.slice(0, take));
-        if (take < target) {
-            report?.(`    ⚠ ${src}: requested ${target}, pool had ${pool.length}`);
-        }
-    }
-    report?.(`  → loaded ${picked.length} seeds across ${sources.length} sources`);
-    return picked;
-}
-const SYSTEM_PROMPT = `You are a postal-address surface-form generator. Given a structured address, produce realistic variants a human might type into a geocoder.
-
-CONSTRAINT — you MUST preserve every kept component value verbatim in the output. You may:
-- vary case (UPPER, lower, Title Case)
-- abbreviate (Saint → St, Avenue → Ave, Boulevard → Blvd, North → N, etc.)
-- vary punctuation (commas, dashes, spaces, line breaks)
-- reorder components (postcode-first, country-first, address-only)
-- drop OPTIONAL components — list them in "dropped"
-
-OPTIONAL components (allowed to drop): country, postcode, dependent_locality, subregion, cedex.
-REQUIRED components (must keep, even if input has them): locality, region (when present),
-street, house_number, venue (when present).
-
-ALWAYS keep AT LEAST 2 components in the final raw text. Single-component variants like
-"VT" or "Paris" are USELESS as eval entries — do not produce them.
-
-You MUST NOT:
-- introduce typos (validator drops these silently — wasted tokens)
-- invent new component values
-- produce text longer than 500 characters
-- output a degenerate single-token answer
-
-OUTPUT a JSON array of N objects, each shaped {"raw": "...", "dropped": ["..."]}.`;
-function buildUserPrompt(seed, n) {
-    return `INPUT:
-${JSON.stringify({ raw: seed.raw, components: seed.components, country: seed.country }, null, 2)}
-
-N: ${n}`;
-}
-function makeDeepseekProvider(model) {
-    const apiKey = $private.DEEPSEEK_API_KEY;
-    if (!apiKey)
-        throw new Error("DEEPSEEK_API_KEY env var is required for provider deepseek");
-    return {
-        name: "deepseek",
-        model,
-        async generateVariants(seed, n) {
-            const res = await fetch("https://api.deepseek.com/chat/completions", {
-                method: "POST",
-                headers: { Authorization: `Bearer ${apiKey}`, "Content-Type": "application/json" },
-                body: JSON.stringify({
-                    model,
-                    messages: [
-                        { role: "system", content: SYSTEM_PROMPT },
-                        { role: "user", content: buildUserPrompt(seed, n) },
-                    ],
-                    response_format: { type: "json_object" },
-                    max_tokens: 800,
-                    temperature: 0.7,
-                }),
-                signal: AbortSignal.timeout(60_000),
-            });
-            if (!res.ok)
-                throw new Error(`DeepSeek HTTP ${res.status}: ${await res.text()}`);
-            const data = (await res.json());
-            const content = data.choices?.[0]?.message.content ?? "{}";
-            return parseCandidates(content);
-        },
-    };
-}
-function makeAnthropicProvider(model) {
-    const apiKey = $private.ANTHROPIC_API_KEY;
-    if (!apiKey)
-        throw new Error("ANTHROPIC_API_KEY env var is required for provider anthropic");
-    return {
-        name: "anthropic",
-        model,
-        async generateVariants(seed, n) {
-            const res = await fetch("https://api.anthropic.com/v1/messages", {
-                method: "POST",
-                headers: {
-                    "x-api-key": apiKey,
-                    "anthropic-version": "2023-06-01",
-                    "Content-Type": "application/json",
-                },
-                body: JSON.stringify({
-                    model,
-                    max_tokens: 800,
-                    system: SYSTEM_PROMPT,
-                    messages: [{ role: "user", content: buildUserPrompt(seed, n) }],
-                }),
-                signal: AbortSignal.timeout(60_000),
-            });
-            if (!res.ok)
-                throw new Error(`Anthropic HTTP ${res.status}: ${await res.text()}`);
-            const data = (await res.json());
-            const text = data.content?.find((c) => c.type === "text")?.text ?? "[]";
-            return parseCandidates(text);
-        },
-    };
-}
-function parseCandidates(text) {
-    // Strip markdown fences the model sometimes wraps around JSON
-    const cleaned = text.replaceAll(/^```(?:json)?\n?|\n?```$/g, "").trim();
-    const parsed = tryParsingJSON(cleaned);
-    if (Array.isArray(parsed))
-        return parsed;
-    // Some providers wrap in {"variants": [...]} or {"candidates": [...]}
-    if (typeof parsed === "object" && parsed !== null) {
-        for (const key of ["variants", "candidates", "results"]) {
-            const v = parsed[key];
-            if (Array.isArray(v))
-                return v;
-        }
-    }
-    return [];
-}
-// ── Validator ─────────────────────────────────────────────────────────────
-function normalize(s) {
-    return s.toLowerCase().replaceAll(/\s+/g, " ").trim();
-}
-/**
- * Components that are NEVER allowed to be dropped — keeps degenerate single-token candidates out.
- */
-const REQUIRED_COMPONENT_TAGS = new Set(["locality", "region", "street", "house_number", "venue"]);
-function validate(seed, candidate) {
-    if (!candidate.raw || typeof candidate.raw !== "string")
-        return false;
-    if (candidate.raw.length > MAX_CANDIDATE_LENGTH)
-        return false;
-    if (/```|<\/?\w+>|^\s*\{/.test(candidate.raw))
-        return false;
-    const normRaw = normalize(candidate.raw);
-    const dropped = new Set(candidate.dropped);
-    // LLM cannot drop required components present in the seed.
-    for (const tag of dropped) {
-        if (REQUIRED_COMPONENT_TAGS.has(tag) && seed.components[tag])
-            return false;
-    }
-    // Every kept component value must appear verbatim (post-normalization) in the candidate raw.
-    let keptCount = 0;
-    for (const [tag, value] of Object.entries(seed.components)) {
-        if (dropped.has(tag))
-            continue;
-        if (!value)
-            continue;
-        if (!normRaw.includes(normalize(value)))
-            return false;
-        keptCount++;
-    }
-    // Reject degenerate single-component candidates ("VT", "Paris" alone).
-    if (keptCount < 2)
-        return false;
-    return true;
-}
-// ── Main pipeline ─────────────────────────────────────────────────────────
-export async function expandGolden(options = {}, report) {
-    const corpusPath = options.corpus ?? dataRootPath("corpus", "versioned", "v0.2.0", "corpus-v0.2.0", "test", "part-0000.parquet");
-    const count = options.count ?? 100;
-    const variants = options.variants ?? 5;
-    const providerName = options.provider ?? "deepseek";
-    const model = options.model ?? (providerName === "anthropic" ? "claude-haiku-4-5-20251001" : "deepseek-chat");
-    const concurrencyLimit = options.concurrency ?? 4;
-    const includeSources = options.includeSources ? new Set(options.includeSources.split(",").map((s) => s.trim())) : null;
-    const ts = new Date().toISOString().replaceAll(/[:.]/g, "-").slice(0, 19);
-    const outputPath = options.output ?? `data/eval/golden/candidates/expand-${ts}.jsonl`;
-    const provider = providerName === "anthropic" ? makeAnthropicProvider(model) : makeDeepseekProvider(model);
-    report?.(`provider: ${provider.name}  model: ${provider.model}`);
-    const seeds = await loadSeeds(corpusPath, count, includeSources, report);
-    if (!seeds.length) {
-        throw new Error("no seeds loaded — corpus path or filter is wrong");
-    }
-    await mkdir(dirname(outputPath), { recursive: true });
-    const outRows = [];
-    let kept = 0;
-    let dropped = 0;
-    let errored = 0;
-    // Bounded-concurrency worker pool
-    let cursor = 0;
-    const workers = Array.from({ length: Math.min(concurrencyLimit, seeds.length) }, async () => {
-        while (true) {
-            const i = cursor++;
-            if (i >= seeds.length)
-                return;
-            const seed = seeds[i];
-            try {
-                const candidates = await provider.generateVariants(seed, variants);
-                for (const cand of candidates) {
-                    if (validate(seed, cand)) {
-                        const goldenCandidate = {
-                            raw: cand.raw,
-                            components: { ...seed.components },
-                            country: seed.country,
-                            source: `expand-golden:${provider.name}`,
-                            seed_source_id: seed.source_id,
-                            seed_source_adapter: seed.source,
-                            dropped_components: cand.dropped ?? [],
-                            provenance: { provider: provider.name, model: provider.model },
-                        };
-                        // Remove dropped components from the components map
-                        for (const tag of goldenCandidate.dropped_components) {
-                            // oxlint-disable-next-line typescript/no-dynamic-delete -- removing one key from a plain record; the object is not on a hot path
-                            delete goldenCandidate.components[tag];
-                        }
-                        outRows.push(goldenCandidate);
-                        kept++;
-                    }
-                    else {
-                        dropped++;
-                    }
-                }
-            }
-            catch (error) {
-                errored++;
-                report?.(`  ✗ seed ${seed.source_id}: ${error.message}`);
-            }
-            if ((i + 1) % 10 === 0) {
-                report?.(`  progress: ${i + 1}/${seeds.length}  kept=${kept}  dropped=${dropped}  errored=${errored}`);
-            }
-        }
-    });
-    await Promise.all(workers);
-    {
-        await using out = createNewlineWriter(outputPath);
-        for (const row of outRows) {
-            await out.write(JSON.stringify(row));
-        }
-    }
-    report?.(`=== summary ===`);
-    report?.(`seeds processed:  ${seeds.length}`);
-    report?.(`candidates kept:  ${kept}`);
-    report?.(`candidates dropped (validator): ${dropped}`);
-    report?.(`seeds with errors: ${errored}`);
-    report?.(`yield: ${seeds.length ? ((kept / (seeds.length * variants)) * 100).toFixed(1) : "0"}%`);
-    report?.(`output:           ${outputPath}`);
-    return { seedsProcessed: seeds.length, kept, dropped, errored, outputPath };
-}
-//# sourceMappingURL=golden-expand.js.map

package/out/src/tools/golden-expand.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"golden-expand.js","sourceRoot":"","sources":["../../../src/tools/golden-expand.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8CG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAA;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAEnC,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAA;AAC/C,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AACpD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEjD,6EAA6E;AAE7E;;GAEG;AACH,MAAM,oBAAoB,GAAG,GAAG,CAAA;AA+EhC,6EAA6E;AAE7E;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,MAAgB,EAAE,MAAgB;IAC3D,MAAM,GAAG,GAA2B,EAAE,CAAA;IACtC,IAAI,UAAU,GAAkB,IAAI,CAAA;IACpC,IAAI,aAAa,GAAa,EAAE,CAAA;IAEhC,MAAM,KAAK,GAAG,GAAG,EAAE;QAClB,IAAI,UAAU,IAAI,aAAa,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;YAChE,GAAG,CAAC,UAAU,CAAC,GAAG,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;QACjD,CAAC;QAED,UAAU,GAAG,IAAI,CAAA;QACjB,aAAa,GAAG,EAAE,CAAA;IACnB,CAAC,CAAA;IAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACxB,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,KAAK,KAAK,GAAG,EAAE,CAAC;YACnB,KAAK,EAAE,CAAA;YAEP,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;QAEzC,IAAI,MAAM,KAAK,GAAG,IAAI,UAAU,KAAK,GAAG,EAAE,CAAC;YAC1C,KAAK,EAAE,CAAA;YACP,UAAU,GAAG,GAAI,CAAA;YACjB,aAAa,GAAG,CAAC,GAAG,CAAC,CAAA;QACtB,CAAC;aAAM,CAAC;YACP,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QACxB,CAAC;IACF,CAAC;IAED,KAAK,EAAE,CAAA;IAEP,OAAO,GAAG,CAAA;AACX,CAAC;AAED,KAAK,UAAU,SAAS,CACvB,UAAkB,EAClB,KAAa,EACb,cAAkC,EAClC,MAA+B;IAE/B,MAAM,KAAK,GAAG,UAAU;SACtB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,OAAO,CAAC,CAAA;IAEjB,MAAM,EAAE,CAAC,sBAAsB,KAAK,CAAC,MAAM,sBAAsB,KAAK,eAAe,CAAC,CAAA;IAEtF,IAAI,cAAc,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,6BAA6B,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAC/E,CAAC;IAED,6FAA6F;IAC7F,8FAA8F;IAC9F,8EAA8E;IAC9E,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;IAC1C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,KAAK,EAAE,IAAI,CAAC,CAAA;IAChD,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,qBAAqB,GAAG,CAAC,CAAA;IAE7B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAA;QAEjC,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,IAAI,EAAE,CAAqB,CAAA;YAErD,IAAI,CAAC,GAAG;gBAAE,MAAK;YAEf,OAAO,EAAE,CAAA;YAET,0FAA0F;YAC1F,IAAI,cAAc,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,SAAQ;YAC/D,MAAM,UAAU,GAAG,gBAAgB,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,EAAE,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAA;YAEvE,4FAA4F;YAC5F,IAAI,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxC,qBAAqB,EAAE,CAAA;gBAEvB,SAAQ;YACT,CAAC;YAED,MAAM,IAAI,GAAS;gBAClB,GAAG,EAAE,GAAG,CAAC,GAAG;gBACZ,UAAU;gBACV,OAAO,EAAE,GAAG,CAAC,OAAO;gBACpB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;aACxB,CAAA;YAED,IAAI,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAErC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACb,MAAM,GAAG,EAAE,CAAA;gBACX,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;YACjC,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBACpC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAClB,CAAC;QACF,CAAC;QAED,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,EAAE,CACP,aAAa,OAAO,gBAAgB,KAAK,CAAC,MAAM,uCAAuC,qBAAqB,EAAE,CAC9G,CAAA;IAED,MAAM,EAAE,CAAC,0BAA0B,CAAC,CAAA;IAEpC,KAAK,MAAM,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACpC,MAAM,EAAE,CAAC,OAAO,GAAG,KAAK,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;IACvC,CAAC;IAED,qFAAqF;IACrF,0FAA0F;IAC1F,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;IACtD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,KAAK,GAAG,SAAS,GAAG,OAAO,CAAC,MAAM,CAAA;IACpD,MAAM,MAAM,GAAW,EAAE,CAAA;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAE,CAAA;QAC/B,MAAM,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAElD,8EAA8E;QAC9E,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC1C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAC5C;YAAA,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAE,EAAE,IAAI,CAAC,CAAC,CAAE,CAAC,CAAA;QAC3C,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAA;QAEnC,IAAI,IAAI,GAAG,MAAM,EAAE,CAAC;YACnB,MAAM,EAAE,CAAC,SAAS,GAAG,eAAe,MAAM,cAAc,IAAI,CAAC,MAAM,EAAE,CAAC,CAAA;QACvE,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,cAAc,MAAM,CAAC,MAAM,iBAAiB,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IAE9E,OAAO,MAAM,CAAA;AACd,CAAC;AAUD,MAAM,aAAa,GAAG;;;;;;;;;;;;;;;;;;;;;;kFAsB4D,CAAA;AAElF,SAAS,eAAe,CAAC,IAAU,EAAE,CAAS;IAC7C,OAAO;EACN,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;;KAE3F,CAAC,EAAE,CAAA;AACR,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAa;IAC1C,MAAM,MAAM,GAAG,QAAQ,CAAC,gBAAgB,CAAA;IAExC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,4DAA4D,CAAC,CAAA;IAE1F,OAAO;QACN,IAAI,EAAE,UAAU;QAChB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,2CAA2C,EAAE;gBACpE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,aAAa,EAAE,UAAU,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;gBAClF,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK;oBACL,QAAQ,EAAE;wBACT,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,aAAa,EAAE;wBAC1C,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE;qBACnD;oBACD,eAAe,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE;oBACxC,UAAU,EAAE,GAAG;oBACf,WAAW,EAAE,GAAG;iBAChB,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACnC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YAChF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA0D,CAAA;YACxF,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,OAAO,IAAI,IAAI,CAAA;YAE1D,OAAO,eAAe,CAAC,OAAO,CAAC,CAAA;QAChC,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAa;IAC3C,MAAM,MAAM,GAAG,QAAQ,CAAC,iBAAiB,CAAA;IAEzC,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAA;IAE5F,OAAO;QACN,IAAI,EAAE,WAAW;QACjB,KAAK;QACL,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,uCAAuC,EAAE;gBAChE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE;oBACR,WAAW,EAAE,MAAM;oBACnB,mBAAmB,EAAE,YAAY;oBACjC,cAAc,EAAE,kBAAkB;iBAClC;gBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;oBACpB,KAAK;oBACL,UAAU,EAAE,GAAG;oBACf,MAAM,EAAE,aAAa;oBACrB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,EAAE,CAAC;iBAC/D,CAAC;gBACF,MAAM,EAAE,WAAW,CAAC,OAAO,CAAC,MAAM,CAAC;aACnC,CAAC,CAAA;YAEF,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAA;YACjF,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAwD,CAAA;YACtF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,EAAE,IAAI,IAAI,IAAI,CAAA;YAEvE,OAAO,eAAe,CAAC,IAAI,CAAC,CAAA;QAC7B,CAAC;KACD,CAAA;AACF,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACpC,8DAA8D;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,2BAA2B,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAA;IAEvE,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;IAEtC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAAE,OAAO,MAAqB,CAAA;IAEvD,sEAAsE;IACtE,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,YAAY,EAAE,SAAS,CAAC,EAAE,CAAC;YACzD,MAAM,CAAC,GAAI,MAAkC,CAAC,GAAG,CAAC,CAAA;YAElD,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBAAE,OAAO,CAAgB,CAAA;QAC9C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,CAAA;AACV,CAAC;AAED,6EAA6E;AAE7E,SAAS,SAAS,CAAC,CAAS;IAC3B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,uBAAuB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,cAAc,EAAE,OAAO,CAAC,CAAC,CAAA;AAElG,SAAS,QAAQ,CAAC,IAAU,EAAE,SAAoB;IACjD,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,OAAO,SAAS,CAAC,GAAG,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IAErE,IAAI,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,oBAAoB;QAAE,OAAO,KAAK,CAAA;IAE7D,IAAI,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAC3D,MAAM,OAAO,GAAG,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAA;IACxC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;IAE1C,2DAA2D;IAC3D,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;QAC3B,IAAI,uBAAuB,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAA;IAC3E,CAAC;IAED,6FAA6F;IAC7F,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC5D,IAAI,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAQ;QAE9B,IAAI,CAAC,KAAK;YAAE,SAAQ;QAEpB,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAAE,OAAO,KAAK,CAAA;QAErD,SAAS,EAAE,CAAA;IACZ,CAAC;IAED,uEAAuE;IACvE,IAAI,SAAS,GAAG,CAAC;QAAE,OAAO,KAAK,CAAA;IAE/B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,6EAA6E;AAE7E,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,UAA+B,EAAE,EACjC,MAA+B;IAE/B,MAAM,UAAU,GACf,OAAO,CAAC,MAAM,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,EAAE,mBAAmB,CAAC,CAAA;IAE9G,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,GAAG,CAAA;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IACtC,MAAM,YAAY,GAAG,OAAO,CAAC,QAAQ,IAAI,UAAU,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,CAAC,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,2BAA2B,CAAC,CAAC,CAAC,eAAe,CAAC,CAAA;IAC7G,MAAM,gBAAgB,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAA;IACjD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,OAAO,CAAC,cAAc,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IACtH,MAAM,EAAE,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACzE,MAAM,UAAU,GAAG,OAAO,CAAC,MAAM,IAAI,sCAAsC,EAAE,QAAQ,CAAA;IAErF,MAAM,QAAQ,GAAG,YAAY,KAAK,WAAW,CAAC,CAAC,CAAC,qBAAqB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,oBAAoB,CAAC,KAAK,CAAC,CAAA;IAC1G,MAAM,EAAE,CAAC,aAAa,QAAQ,CAAC,IAAI,YAAY,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAA;IAEhE,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,cAAc,EAAE,MAAM,CAAC,CAAA;IAExE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,kDAAkD,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,MAAM,OAAO,GAAsB,EAAE,CAAA;IACrC,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,OAAO,GAAG,CAAC,CAAA;IACf,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,kCAAkC;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,gBAAgB,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC3F,OAAO,IAAI,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,MAAM,EAAE,CAAA;YAElB,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM;gBAAE,OAAM;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;YAEtB,IAAI,CAAC;gBACJ,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,gBAAgB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;gBAElE,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;oBAC/B,IAAI,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC;wBAC1B,MAAM,eAAe,GAAoB;4BACxC,GAAG,EAAE,IAAI,CAAC,GAAG;4BACb,UAAU,EAAE,EAAE,GAAG,IAAI,CAAC,UAAU,EAAE;4BAClC,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,iBAAiB,QAAQ,CAAC,IAAI,EAAE;4BACxC,cAAc,EAAE,IAAI,CAAC,SAAS;4BAC9B,mBAAmB,EAAE,IAAI,CAAC,MAAM;4BAChC,kBAAkB,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;4BACtC,UAAU,EAAE,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE;yBAC9D,CAAA;wBAED,oDAAoD;wBACpD,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,kBAAkB,EAAE,CAAC;4BACtD,iIAAiI;4BACjI,OAAO,eAAe,CAAC,UAAU,CAAC,GAAG,CAAC,CAAA;wBACvC,CAAC;wBAED,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;wBAE7B,IAAI,EAAE,CAAA;oBACP,CAAC;yBAAM,CAAC;wBACP,OAAO,EAAE,CAAA;oBACV,CAAC;gBACF,CAAC;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAA;gBACT,MAAM,EAAE,CAAC,YAAY,IAAI,CAAC,SAAS,KAAM,KAAe,CAAC,OAAO,EAAE,CAAC,CAAA;YACpE,CAAC;YAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,UAAU,IAAI,aAAa,OAAO,aAAa,OAAO,EAAE,CAAC,CAAA;YACvG,CAAC;QACF,CAAC;IACF,CAAC,CAAC,CAAA;IAEF,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;IAE1B,CAAC;QACA,YAAY,GAAG,GAAG,mBAAmB,CAAC,UAAU,CAAC,CAAA;QAEjD,KAAK,MAAM,GAAG,IAAI,OAAO,EAAE,CAAC;YAC3B,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAA;QACrC,CAAC;IACF,CAAC;IAED,MAAM,EAAE,CAAC,iBAAiB,CAAC,CAAA;IAC3B,MAAM,EAAE,CAAC,qBAAqB,KAAK,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7C,MAAM,EAAE,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAA;IACrC,MAAM,EAAE,CAAC,mCAAmC,OAAO,EAAE,CAAC,CAAA;IACtD,MAAM,EAAE,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAA;IACzC,MAAM,EAAE,CAAC,UAAU,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAA;IACjG,MAAM,EAAE,CAAC,qBAAqB,UAAU,EAAE,CAAC,CAAA;IAE3C,OAAO,EAAE,cAAc,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,CAAA;AAC5E,CAAC"}

package/out/src/tools/golden-promote.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"golden-promote.d.ts","sourceRoot":"","sources":["../../../src/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAqCH,MAAM,WAAW,YAAY;IAC5B,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,UAAU,EAAE,MAAM,CAAA;KAAE,CAAA;IAClH,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAClC;AAED,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB;;OAEG;IACH,SAAS,CAAC,EAAE,OAAO,CAAA;IACnB;;OAEG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAyED,wBAAsB,aAAa,CAClC,OAAO,EAAE,oBAAoB,EAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,YAAY,CAAC,CAgLvB"}

package/out/src/tools/golden-promote.js

@@ -1,244 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
- *   human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
- *
- *   ## What it does
- *
- *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
- *   2. Reads the previous-version golden dir for forward-copy + dedup base
- *   3. Applies filters that drop candidates unlikely to be human-typed:
- *
- *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
- *                 before locality only)
- *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
- *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
- *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
- *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
- *   7. Writes MANIFEST.json with sha256 of each output file
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus golden promote \
- *   --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
- *   --bump-to v0.1.1 \
- *   --prior v0.1.0
- *   ```
- */
-import { copyFileSync, existsSync, mkdirSync, readdirSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-import { sha256File } from "@mailwoman/core/utils";
-import { createNewlineWriter, JSONSpliterator } from "spliterator";
-// ── Types ──────────────────────────────────────────────────────────────────
-/**
- * Components a candidate needs before it is rich enough to be worth a golden-set slot.
- */
-const MIN_GOLDEN_COMPONENTS = 5;
-/**
- * Components an existing entry must still carry to survive promotion.
- */
-const MIN_PROMOTABLE_COMPONENTS = 4;
-/**
- * Quote-delimited segments above which the raw line is malformed rather than merely quoted.
- */
-const MAX_QUOTE_SEGMENTS = 3;
-// ── Filters ───────────────────────────────────────────────────────────────
-function normalize(s) {
-    return s.toLowerCase().replaceAll(/\s+/g, " ").trim();
-}
-/**
- * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
- * together rather than human-typed.
- */
-function isComponentsGlued(entry) {
-    const componentCount = Object.keys(entry.components).length;
-    if (componentCount < MIN_GOLDEN_COMPONENTS)
-        return false;
-    const separators = (entry.raw.match(/[,\n;]/g) ?? []).length;
-    return separators < 2;
-}
-/**
- * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
- * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
- * often precedes locality there).
- */
-function isPostcodeBadlyLeading(entry) {
-    if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS)
-        return false;
-    if (entry.country === "FR" || entry.country === "France")
-        return false;
-    const postcode = entry.components.postcode;
-    if (!postcode)
-        return false;
-    const postcodeIdx = entry.raw.indexOf(postcode);
-    if (postcodeIdx === -1)
-        return false;
-    return postcodeIdx < entry.raw.length / 3;
-}
-/**
- * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
- * suggests the LLM emitted markup instead of an address.
- */
-function isSuspicious(entry) {
-    const raw = entry.raw;
-    // eslint-disable-next-line no-control-regex
-    if (/[\u0000-\u001F\u007F]/.test(raw))
-        return true;
-    const openBrackets = (raw.match(/[[({<]/g) ?? []).length;
-    const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length;
-    if (openBrackets !== closeBrackets)
-        return true;
-    if (raw.split('"').length > MAX_QUOTE_SEGMENTS)
-        return true;
-    // too many quote marks
-    return false;
-}
-// ── IO ─────────────────────────────────────────────────────────────────────
-/**
- * Read a JSONL, tolerating a missing file (returns `[]`) — prior golden dirs may lack a bucket.
- */
-async function readJSONLIfPresent(path) {
-    if (!existsSync(path))
-        return [];
-    return Array.fromAsync(JSONSpliterator.fromAsync(path));
-}
-// ── Main ───────────────────────────────────────────────────────────────────
-export async function promoteGolden(options, report) {
-    const prior = options.prior ?? "v0.1.0";
-    const goldenRoot = options.goldenRoot ?? "data/eval/golden";
-    const applyFilters = !options.noFilters;
-    const dryRun = options.dryRun ?? false;
-    report?.(`reading candidates: ${options.input}`);
-    const candidates = await readJSONLIfPresent(options.input);
-    report?.(`  ${candidates.length} candidates loaded`);
-    // Forward-copy base: existing entries from the prior golden version go forward verbatim,
-    // and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
-    const priorDir = join(goldenRoot, prior);
-    const priorEntries = [];
-    const seenNormalized = new Set();
-    if (existsSync(priorDir)) {
-        for (const f of readdirSync(priorDir).filter((n) => n.endsWith(".jsonl"))) {
-            const country = f.replace(".jsonl", "").toUpperCase();
-            const entries = await readJSONLIfPresent(join(priorDir, f));
-            priorEntries.push({ country, entries });
-            for (const e of entries) {
-                seenNormalized.add(normalize(e.raw));
-            }
-            report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`);
-        }
-    }
-    else {
-        report?.(`  ⚠ prior dir ${priorDir} not found — starting fresh`);
-    }
-    // Filter pass
-    const stats = {
-        candidatesIn: candidates.length,
-        filteredOut: { glued: 0, postcodeLeading: 0, suspicious: 0, duplicate: 0, forwardDup: 0 },
-        kept: 0,
-        perCountry: {},
-    };
-    const accepted = [];
-    const seenInBatch = new Set();
-    for (const cand of candidates) {
-        const norm = normalize(cand.raw);
-        // Dedup pass 1: against prior versioned golden
-        if (seenNormalized.has(norm)) {
-            stats.filteredOut.forwardDup++;
-            continue;
-        }
-        // Dedup pass 2: against this batch
-        if (seenInBatch.has(norm)) {
-            stats.filteredOut.duplicate++;
-            continue;
-        }
-        if (applyFilters) {
-            if (isComponentsGlued(cand)) {
-                stats.filteredOut.glued++;
-                continue;
-            }
-            if (isPostcodeBadlyLeading(cand)) {
-                stats.filteredOut.postcodeLeading++;
-                continue;
-            }
-            if (isSuspicious(cand)) {
-                stats.filteredOut.suspicious++;
-                continue;
-            }
-        }
-        accepted.push(cand);
-        seenInBatch.add(norm);
-        stats.kept++;
-        const country = cand.country || "OTHER";
-        stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1;
-    }
-    // Bucket per country, including forward-copied entries
-    const buckets = new Map();
-    for (const { country, entries } of priorEntries) {
-        // Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
-        buckets.set(country, [...entries]);
-    }
-    for (const cand of accepted) {
-        const key = (cand.country || "OTHER").toUpperCase();
-        if (!buckets.has(key)) {
-            buckets.set(key, []);
-        }
-        buckets.get(key).push(cand);
-    }
-    // Output
-    const outDir = join(goldenRoot, options.bumpTo);
-    report?.(`=== plan ===`);
-    report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`);
-    for (const [key, entries] of buckets) {
-        report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`);
-    }
-    report?.(`=== filter stats ===`);
-    report?.(`candidates in:        ${stats.candidatesIn}`);
-    report?.(`  filtered (glued):           ${stats.filteredOut.glued}`);
-    report?.(`  filtered (postcode-lead):   ${stats.filteredOut.postcodeLeading}`);
-    report?.(`  filtered (suspicious):      ${stats.filteredOut.suspicious}`);
-    report?.(`  filtered (dup-in-batch):    ${stats.filteredOut.duplicate}`);
-    report?.(`  filtered (dup-vs-prior):    ${stats.filteredOut.forwardDup}`);
-    report?.(`  kept:                       ${stats.kept}`);
-    report?.(`per-country (kept):`);
-    for (const [c, n] of Object.entries(stats.perCountry).toSorted((a, b) => b[1] - a[1])) {
-        report?.(`  ${c}: ${n}`);
-    }
-    if (dryRun) {
-        report?.(`(dry-run: no files written)`);
-        return stats;
-    }
-    mkdirSync(outDir, { recursive: true });
-    const manifest = {
-        promoted_at: new Date().toISOString(),
-        from: options.input,
-        from_sha256: await sha256File(options.input),
-        files: {},
-    };
-    for (const [key, entries] of buckets) {
-        const filename = `${key.toLowerCase()}.jsonl`;
-        const path = join(outDir, filename);
-        {
-            await using out = createNewlineWriter(path);
-            for (const entry of entries) {
-                await out.write(JSON.stringify(entry));
-            }
-        }
-        manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) };
-    }
-    // Forward-copy non-.jsonl files (README.md, etc.) from prior
-    if (existsSync(priorDir)) {
-        for (const f of readdirSync(priorDir).filter((n) => !n.endsWith(".jsonl"))) {
-            copyFileSync(join(priorDir, f), join(outDir, f));
-            report?.(`  forward-copied: ${f}`);
-        }
-    }
-    writeFileSync(join(outDir, "MANIFEST.json"), JSON.stringify(manifest, null, 2) + "\n");
-    report?.(`✓ promoted to ${outDir}`);
-    return stats;
-}
-//# sourceMappingURL=golden-promote.js.map