@mailwoman/corpus 9.3.0 → 10.0.0

package/out/tools/golden-promote.js

@@ -1,240 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Promote LLM-synthesized golden-set candidates into a versioned golden dir, with
- *   human-typed-likelihood filters + dedup. Companion to `golden-expand.ts`.
- *
- *   ## What it does
- *
- *   1. Reads a candidates JSONL from `data/eval/golden/candidates/`
- *   2. Reads the previous-version golden dir for forward-copy + dedup base
- *   3. Applies filters that drop candidates unlikely to be human-typed:
- *
- *        - Components-glued-without-commas (5+ components but <2 separators → freeform jumble)
- *        - Postcode-at-start with many other components (US/UK conventions put postcode last; FR puts it
- *                 before locality only)
- *        - Suspicious-token signals (unmatched brackets, control chars, etc.)
- *   4. Dedupes by normalized raw (case-insensitive, whitespace-collapsed)
- *   5. Splits by country (US/FR/other) and writes `data/eval/golden/v<X.Y.Z>/{us,fr,other}.jsonl`
- *   6. Forward-copies the prior version's adversarial.jsonl + README as-is
- *   7. Writes MANIFEST.json with sha256 of each output file
- *
- *   ## Usage
- *
- *   ```sh
- *   mailwoman corpus golden promote \
- *   --input data/eval/golden/candidates/expand-20260518-162627.jsonl \
- *   --bump-to v0.1.1 \
- *   --prior v0.1.0
- *   ```
- */
-import { readDirectory, pathExists } from "@mailwoman/core/fs/readers";
-import { copyFileTo, makeDirectories, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { foldCaseWhitespace } from "@mailwoman/normalize/fold";
-import { join } from "path-ts";
-import { createNewlineWriter, JSONSpliterator } from "spliterator";
-// ── Types ──────────────────────────────────────────────────────────────────
-/**
- * Components a candidate needs before it is rich enough to be worth a golden-set slot.
- */
-const MIN_GOLDEN_COMPONENTS = 5;
-/**
- * Components an existing entry must still carry to survive promotion.
- */
-const MIN_PROMOTABLE_COMPONENTS = 4;
-/**
- * Quote-delimited segments above which the raw line is malformed rather than merely quoted.
- */
-const MAX_QUOTE_SEGMENTS = 3;
-// ── Filters ───────────────────────────────────────────────────────────────
-/**
- * Heuristic: an address with 5+ components but fewer than 2 separators (commas/newlines/dashes) is most likely glued
- * together rather than human-typed.
- */
-function isComponentsGlued(entry) {
-    const componentCount = Object.keys(entry.components).length;
-    if (componentCount < MIN_GOLDEN_COMPONENTS)
-        return false;
-    const separators = (entry.raw.match(/[,\n;]/g) ?? []).length;
-    return separators < 2;
-}
-/**
- * Heuristic: in US/UK conventions, postcode goes at the END of the address. If postcode appears in the first third of a
- * multi-component raw AND there are 4+ components, the LLM probably over-aggressively reordered. FR is exempt (postcode
- * often precedes locality there).
- */
-function isPostcodeBadlyLeading(entry) {
-    if (Object.keys(entry.components).length < MIN_PROMOTABLE_COMPONENTS)
-        return false;
-    if (entry.country === "FR" || entry.country === "France")
-        return false;
-    const postcode = entry.components.postcode;
-    if (!postcode)
-        return false;
-    const postcodeIdx = entry.raw.indexOf(postcode);
-    if (postcodeIdx === -1)
-        return false;
-    return postcodeIdx < entry.raw.length / 3;
-}
-/**
- * Heuristic: catch-all for visually-bad outputs — unmatched brackets, control chars, suspicious punctuation that
- * suggests the LLM emitted markup instead of an address.
- */
-function isSuspicious(entry) {
-    const raw = entry.raw;
-    // eslint-disable-next-line no-control-regex
-    if (/[\u0000-\u001F\u007F]/.test(raw))
-        return true;
-    const openBrackets = (raw.match(/[[({<]/g) ?? []).length;
-    const closeBrackets = (raw.match(/[\])}>]/g) ?? []).length;
-    if (openBrackets !== closeBrackets)
-        return true;
-    if (raw.split('"').length > MAX_QUOTE_SEGMENTS)
-        return true;
-    // too many quote marks
-    return false;
-}
-// ── Main ───────────────────────────────────────────────────────────────────
-export async function promoteGolden(options, report) {
-    const prior = options.prior ?? "v0.1.0";
-    const goldenRoot = options.goldenRoot ?? "data/eval/golden";
-    const applyFilters = !options.noFilters;
-    const dryRun = options.dryRun ?? false;
-    report?.(`reading candidates: ${options.input}`);
-    // The candidates file is the REQUIRED input, so its absence must not read as "zero candidates" —
-    // that promotes the prior version unchanged and reports success. `JSONSpliterator.fromAsync`
-    // reports a missing path as a bare `TypeError` naming its own internals, so name the path here.
-    if (!(await pathExists(options.input))) {
-        throw new Error(`Candidates file not found: ${options.input}`);
-    }
-    const candidates = await Array.fromAsync(JSONSpliterator.fromAsync(options.input));
-    report?.(`  ${candidates.length} candidates loaded`);
-    // Forward-copy base: existing entries from the prior golden version go forward verbatim,
-    // and we dedupe new candidates against them so v_new = v_old ∪ accepted_candidates.
-    const priorDir = join(goldenRoot, prior);
-    const priorEntries = [];
-    const seenNormalized = new Set();
-    if (await pathExists(priorDir)) {
-        for (const f of (await readDirectory(priorDir)).filter((n) => n.endsWith(".jsonl"))) {
-            const country = f.replace(".jsonl", "").toUpperCase();
-            const entries = await Array.fromAsync(JSONSpliterator.fromAsync(join(priorDir, f)));
-            priorEntries.push({ country, entries });
-            for (const e of entries) {
-                seenNormalized.add(foldCaseWhitespace(e.raw));
-            }
-            report?.(`  prior ${country}: ${entries.length} entries (forward-copy base)`);
-        }
-    }
-    else {
-        report?.(`  ⚠ prior dir ${priorDir} not found — starting fresh`);
-    }
-    // Filter pass
-    const stats = {
-        candidatesIn: candidates.length,
-        filteredOut: { glued: 0, postcodeLeading: 0, suspicious: 0, duplicate: 0, forwardDup: 0 },
-        kept: 0,
-        perCountry: {},
-    };
-    const accepted = [];
-    const seenInBatch = new Set();
-    for (const cand of candidates) {
-        const norm = foldCaseWhitespace(cand.raw);
-        // Dedup pass 1: against prior versioned golden
-        if (seenNormalized.has(norm)) {
-            stats.filteredOut.forwardDup++;
-            continue;
-        }
-        // Dedup pass 2: against this batch
-        if (seenInBatch.has(norm)) {
-            stats.filteredOut.duplicate++;
-            continue;
-        }
-        if (applyFilters) {
-            if (isComponentsGlued(cand)) {
-                stats.filteredOut.glued++;
-                continue;
-            }
-            if (isPostcodeBadlyLeading(cand)) {
-                stats.filteredOut.postcodeLeading++;
-                continue;
-            }
-            if (isSuspicious(cand)) {
-                stats.filteredOut.suspicious++;
-                continue;
-            }
-        }
-        accepted.push(cand);
-        seenInBatch.add(norm);
-        stats.kept++;
-        const country = cand.country || "OTHER";
-        stats.perCountry[country] = (stats.perCountry[country] ?? 0) + 1;
-    }
-    // Bucket per country, including forward-copied entries
-    const buckets = new Map();
-    for (const { country, entries } of priorEntries) {
-        // Existing files keyed by filename uppercase (us.jsonl → US, adversarial.jsonl → ADVERSARIAL)
-        buckets.set(country, [...entries]);
-    }
-    for (const cand of accepted) {
-        const key = (cand.country || "OTHER").toUpperCase();
-        if (!buckets.has(key)) {
-            buckets.set(key, []);
-        }
-        buckets.get(key).push(cand);
-    }
-    // Output
-    const outDir = join(goldenRoot, options.bumpTo);
-    report?.(`=== plan ===`);
-    report?.(`output dir: ${outDir}${dryRun ? " (dry-run)" : ""}`);
-    for (const [key, entries] of buckets) {
-        report?.(`  ${key.toLowerCase()}.jsonl: ${entries.length} entries`);
-    }
-    report?.(`=== filter stats ===`);
-    report?.(`candidates in:        ${stats.candidatesIn}`);
-    report?.(`  filtered (glued):           ${stats.filteredOut.glued}`);
-    report?.(`  filtered (postcode-lead):   ${stats.filteredOut.postcodeLeading}`);
-    report?.(`  filtered (suspicious):      ${stats.filteredOut.suspicious}`);
-    report?.(`  filtered (dup-in-batch):    ${stats.filteredOut.duplicate}`);
-    report?.(`  filtered (dup-vs-prior):    ${stats.filteredOut.forwardDup}`);
-    report?.(`  kept:                       ${stats.kept}`);
-    report?.(`per-country (kept):`);
-    for (const [c, n] of Object.entries(stats.perCountry).toSorted((a, b) => b[1] - a[1])) {
-        report?.(`  ${c}: ${n}`);
-    }
-    if (dryRun) {
-        report?.(`(dry-run: no files written)`);
-        return stats;
-    }
-    await makeDirectories(outDir);
-    const manifest = {
-        promoted_at: new Date().toISOString(),
-        from: options.input,
-        from_sha256: await sha256File(options.input),
-        files: {},
-    };
-    for (const [key, entries] of buckets) {
-        const filename = `${key.toLowerCase()}.jsonl`;
-        const path = join(outDir, filename);
-        {
-            await using out = createNewlineWriter(path);
-            for (const entry of entries) {
-                await out.write(JSON.stringify(entry));
-            }
-        }
-        manifest.files[filename] = { entries: entries.length, sha256: await sha256File(path) };
-    }
-    // Forward-copy non-.jsonl files (README.md, etc.) from prior
-    if (await pathExists(priorDir)) {
-        for (const f of (await readDirectory(priorDir)).filter((n) => !n.endsWith(".jsonl"))) {
-            await copyFileTo(join(priorDir, f), join(outDir, f));
-            report?.(`  forward-copied: ${f}`);
-        }
-    }
-    await writeLocalJSONFile(manifest, join(outDir, "MANIFEST.json"));
-    report?.(`✓ promoted to ${outDir}`);
-    return stats;
-}
-//# sourceMappingURL=golden-promote.js.map

package/out/tools/golden-promote.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"golden-promote.js","sourceRoot":"","sources":["../../lib/tools/golden-promote.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,OAAO,EAAE,aAAa,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AACtE,OAAO,EAAE,UAAU,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC5F,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,kBAAkB,EAAE,MAAM,2BAA2B,CAAA;AAC9D,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAIlE,8EAA8E;AAE9E;;GAEG;AACH,MAAM,qBAAqB,GAAG,CAAC,CAAA;AAE/B;;GAEG;AACH,MAAM,yBAAyB,GAAG,CAAC,CAAA;AAEnC;;GAEG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAoC5B,6EAA6E;AAE7E;;;GAGG;AACH,SAAS,iBAAiB,CAAC,KAAkB;IAC5C,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,CAAA;IAE3D,IAAI,cAAc,GAAG,qBAAqB;QAAE,OAAO,KAAK,CAAA;IACxD,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE5D,OAAO,UAAU,GAAG,CAAC,CAAA;AACtB,CAAC;AAED;;;;GAIG;AACH,SAAS,sBAAsB,CAAC,KAAkB;IACjD,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,MAAM,GAAG,yBAAyB;QAAE,OAAO,KAAK,CAAA;IAElF,IAAI,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAA;IACtE,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAA;IAE1C,IAAI,CAAC,QAAQ;QAAE,OAAO,KAAK,CAAA;IAC3B,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;IAE/C,IAAI,WAAW,KAAK,CAAC,CAAC;QAAE,OAAO,KAAK,CAAA;IAEpC,OAAO,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAA;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,KAAkB;IACvC,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,CAAA;IAErB,4CAA4C;IAC5C,IAAI,uBAAuB,CAAC,IAAI,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAClD,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IACxD,MAAM,aAAa,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAA;IAE1D,IAAI,YAAY,KAAK,aAAa;QAAE,OAAO,IAAI,CAAA;IAE/C,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,GAAG,kBAAkB;QAAE,OAAO,IAAI,CAAA;IAE3D,uBAAuB;IACvB,OAAO,KAAK,CAAA;AACb,CAAC;AAED,8EAA8E;AAE9E,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,OAA6B,EAC7B,MAA+B;IAE/B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAA;IACvC,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAA;IAC3D,MAAM,YAAY,GAAG,CAAC,OAAO,CAAC,SAAS,CAAA;IACvC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,KAAK,CAAA;IAEtC,MAAM,EAAE,CAAC,uBAAuB,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEhD,iGAAiG;IACjG,6FAA6F;IAC7F,gGAAgG;IAChG,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CAAC,8BAA8B,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAC/D,CAAC;IAED,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAc,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IAC/F,MAAM,EAAE,CAAC,KAAK,UAAU,CAAC,MAAM,oBAAoB,CAAC,CAAA;IAEpD,yFAAyF;IACzF,oFAAoF;IACpF,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAU,EAAE,KAAK,CAAC,CAAA;IACxC,MAAM,YAAY,GAAkD,EAAE,CAAA;IACtE,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAA;IAExC,IAAI,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACrF,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;YACrD,MAAM,OAAO,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAc,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAChG,YAAY,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;YAEvC,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACzB,cAAc,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;YAC9C,CAAC;YAED,MAAM,EAAE,CAAC,WAAW,OAAO,KAAK,OAAO,CAAC,MAAM,8BAA8B,CAAC,CAAA;QAC9E,CAAC;IACF,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,iBAAiB,QAAQ,6BAA6B,CAAC,CAAA;IACjE,CAAC;IAED,cAAc;IACd,MAAM,KAAK,GAAiB;QAC3B,YAAY,EAAE,UAAU,CAAC,MAAM;QAC/B,WAAW,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,eAAe,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE;QACzF,IAAI,EAAE,CAAC;QACP,UAAU,EAAE,EAAE;KACd,CAAA;IAED,MAAM,QAAQ,GAAkB,EAAE,CAAA;IAClC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAA;IAErC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,MAAM,IAAI,GAAG,kBAAkB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAEzC,+CAA+C;QAC/C,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9B,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;YAE9B,SAAQ;QACT,CAAC;QAED,mCAAmC;QACnC,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAA;YAE7B,SAAQ;QACT,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YAClB,IAAI,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAA;gBAEzB,SAAQ;YACT,CAAC;YAED,IAAI,sBAAsB,CAAC,IAAI,CAAC,EAAE,CAAC;gBAClC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAA;gBAEnC,SAAQ;YACT,CAAC;YAED,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;gBACxB,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAA;gBAE9B,SAAQ;YACT,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnB,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAErB,KAAK,CAAC,IAAI,EAAE,CAAA;QACZ,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,OAAO,CAAA;QACvC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACjE,CAAC;IAED,uDAAuD;IACvD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAA;IAEhD,KAAK,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,YAAY,EAAE,CAAC;QACjD,8FAA8F;QAC9F,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,GAAG,OAAO,CAAC,CAAC,CAAA;IACnC,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,CAAC,WAAW,EAAE,CAAA;QAEnD,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QACrB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAC7B,CAAC;IAED,SAAS;IACT,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAA;IAC/C,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACxB,MAAM,EAAE,CAAC,eAAe,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAE9D,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,EAAE,CAAC,KAAK,GAAG,CAAC,WAAW,EAAE,WAAW,OAAO,CAAC,MAAM,UAAU,CAAC,CAAA;IACpE,CAAC;IAED,MAAM,EAAE,CAAC,sBAAsB,CAAC,CAAA;IAChC,MAAM,EAAE,CAAC,yBAAyB,KAAK,CAAC,YAAY,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC,CAAA;IACpE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC,CAAA;IAC9E,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,CAAA;IACxE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAA;IACzE,MAAM,EAAE,CAAC,iCAAiC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;IACvD,MAAM,EAAE,CAAC,qBAAqB,CAAC,CAAA;IAE/B,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvF,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACzB,CAAC;IAED,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,EAAE,CAAC,6BAA6B,CAAC,CAAA;QAEvC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;IAE7B,MAAM,QAAQ,GAKV;QACH,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACrC,IAAI,EAAE,OAAO,CAAC,KAAK;QACnB,WAAW,EAAE,MAAM,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC;QAC5C,KAAK,EAAE,EAAE;KACT,CAAA;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,OAAO,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,GAAG,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAA;QAC7C,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAA;QAEnC,CAAC;YACA,YAAY,GAAG,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAA;YAE3C,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC7B,MAAM,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAA;YACvC,CAAC;QACF,CAAC;QAED,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,UAAU,CAAC,IAAI,CAAC,EAAE,CAAA;IACvF,CAAC;IAED,6DAA6D;IAC7D,IAAI,MAAM,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACtF,MAAM,UAAU,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;YACpD,MAAM,EAAE,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC;IACF,CAAC;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC,CAAA;IACjE,MAAM,EAAE,CAAC,iBAAiB,MAAM,EAAE,CAAC,CAAA;IAEnC,OAAO,KAAK,CAAA;AACb,CAAC"}

package/out/tools/golden-relabel-street.d.ts

@@ -1,196 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Golden-set street-suffix relabel — v0.1.2 → v0.1.3.
- *
- *   ## Why this exists
- *
- *   The golden answer key and the training corpus disagreed about ONE thing, and the v9.0.0
- *   promotion eval read the disagreement as a model regression (`us.street` 87.4 vs a floor of
- *   87.8). The corpus SPLITS a US street into `street` + `street_suffix` — TIGER's adapter
- *   decomposes at `corpus/src/adapters/tiger/street-decompose.ts`, the `street-affix` slice recipe
- *   teaches it from USPS Pub-28, and `ComponentTag` carries `street_suffix` as a first-class tag.
- *   The golden set FOLDED it: 2,216 US rows carry a `street`, and exactly 2 of them label a
- *   `street_suffix`. Operator ruling, 2026-08-06: **the split is canonical**; the golden is the
- *   stale side. This tool moves the answer key onto the corpus convention.
- *
- *   ## The instrument
- *
- *   `matchTrailingSuffix` from `@mailwoman/codex/us` — the USPS Pub-28 Appendix C table, which is
- *   also what the corpus slice recipe splits on. The table is NOT re-implemented here, and the
- *   libpostal dictionary TIGER reads is deliberately not used: measured on this golden set the two
- *   disagree on 51 US rows, and the disagreements run in the codex table's favour (libpostal's
- *   `directionals.txt` lists `center|c`, so TIGER reads the `C` of "C STREET" as a directional
- *   prefix and then emits no suffix at all).
- *
- *   ## What it changes, and what it refuses to
- *
- *   Applied only to rows whose `country` is `US`. Three branches, mirroring the SHAPE of TIGER's
- *   `decomposeStreet` on codex tables:
- *
- *   - **street type** — the last whitespace-separated word is a Pub-28 suffix, and something is left
- *       over: "Main St" → `street: "Main"`, `street_suffix: "St"` (1,559 rows).
- *   - **street type + post-directional** — the last word is a directional AND the one before it is a
- *       Pub-28 suffix: "Pennsylvania Avenue NW" → `street: "Pennsylvania"`,
- *       `street_suffix: "Avenue NW"` (347 rows). The post-directional joins the suffix rather than
- *       becoming a tag of its own, because that is what the corpus adapter emits; there is no
- *       `street_postfix` tag to move it to.
- *   - **everything else is left folded** and reported. In particular a BARE post-directional tail
- *       ("Seymour East", "BROADWAY N" — 16 rows) is NOT split: a directional is not a Pub-28 suffix,
- *       and the observed rows in that class are unit-contaminated ("1ST AVE SW BOX E", where the
- *       trailing "E" is a box letter).
- *
- *   FR rows are untouched, deliberately and permanently as far as this tool is concerned. French
- *   street typology puts the type FIRST ("Rue de la Paix") and the golden labels only 7 of 665 FR
- *   street rows with a `street_prefix`; whether FR should split at all is a different question with
- *   a different table behind it, and nothing here should be read as having answered it.
- *
- *   ## Surface bytes
- *
- *   The split is a break at a whitespace run in the ORIGINAL string — no trimming, no case
- *   normalization, no re-joining of tokens. `street + gap + street_suffix` reconstructs the input
- *   byte-for-byte, so the whitespace between them belongs to neither span (the same shape the corpus
- *   adapter's spans have). The tool asserts this per row and refuses to write a file if it ever
- *   fails.
- */
-/**
- * A golden-set row, as stored one-per-line in `us.jsonl` / `fr.jsonl` / `adversarial.jsonl`. Only the fields this tool
- * reads are modeled; every other key rides through untouched.
- */
-export interface GoldenStreetRow {
-    raw: string;
-    components: Record<string, string>;
-    country?: string;
-    source?: string;
-    notes?: string;
-    [key: string]: unknown;
-}
-/**
- * What the tool decided about one row. Every value except the two `split-*` classes means "left folded".
- */
-export type GoldenRelabelClass = "split-suffix" | "split-suffix-postdirectional" | "split-prefix-only" | "already-split" | "single-token" | "suffix-only-street" | "postdirectional-tail-only" | "no-suffix-match" | "no-street" | "not-us" | "untrimmed-street";
-/**
- * A review trigger on a row the tool DID change. A flag is never an adjudication — it marks the row for the operator's
- * deck, and the split is applied either way.
- */
-export interface GoldenRelabelFlag {
-    kind: "name-prone-suffix" | "venue-context" | "remainder-is-affix";
-    detail: string;
-}
-/**
- * Row-level relabel outcome.
- */
-export interface GoldenRelabelResult {
-    /**
-     * The row to write. Identical object reference when nothing changed.
-     */
-    row: GoldenStreetRow;
-    changed: boolean;
-    rowClass: GoldenRelabelClass;
-    flags: GoldenRelabelFlag[];
-    /**
-     * A leading directional was lifted into `street_prefix` on this row.
-     */
-    prefixSplit: boolean;
-    /**
-     * The street span as it stood in the parent version — recorded for the review deck.
-     */
-    beforeStreet?: string;
-}
-/**
- * Options for {@linkcode relabelGoldenStreetRow}.
- */
-export interface RelabelStreetRowOptions {
-    /**
-     * Also lift a folded LEADING directional out into `street_prefix`. Default true.
-     *
-     * On by default because the fold applies both ways and the answer key has to be corrected on both, or the correction
-     * is not a correction: 207 of the 1,682 split dev rows (12.3%) still opened with a directional after the suffix move
-     * — "N Desmet Avenue" would have graded `street: "N Desmet"` against a model that says `street_prefix: "N", street:
-     * "Desmet"`. Turn it OFF only to measure what the prefix fold alone costs.
-     */
-    splitPrefix?: boolean;
-}
-/**
- * Decide, and apply, the US street-span split for ONE golden row. Pure: never mutates its argument, and returns the
- * same object reference when the row is left alone.
- */
-export declare function relabelGoldenStreetRow(row: GoldenStreetRow, options?: RelabelStreetRowOptions): GoldenRelabelResult;
-/**
- * Per-class row counts for one relabelled file.
- */
-export type GoldenRelabelCounts = Record<GoldenRelabelClass, number>;
-/**
- * One line of the review deck: what a changed (or notably unchanged) row looked like before and after.
- */
-export interface GoldenRelabelDeckEntry {
-    file: string;
-    line: number;
-    raw: string;
-    rowClass: GoldenRelabelClass;
-    before: Record<string, string>;
-    after: Record<string, string>;
-    flags: GoldenRelabelFlag[];
-}
-/**
- * Options for {@linkcode relabelGoldenDirectory}.
- */
-export interface RelabelGoldenOptions {
-    /**
-     * Parent golden version dir (read-only), e.g. `data/eval/golden/v0.1.2`.
-     */
-    input: string;
-    /**
-     * Output golden version dir. Created; never overwritten in place.
-     */
-    output: string;
-    /**
-     * Review-deck JSONL path. Default `<output>/REVIEW-DECK.jsonl`.
-     */
-    deck?: string;
-    /**
-     * Parent version label recorded in the manifest. Default: the input dir's basename.
-     */
-    parentLabel?: string;
-    /**
-     * Tool provenance recorded in the manifest — the commit the relabel ran at.
-     */
-    commit?: string;
-    /**
-     * Passed through to {@linkcode relabelGoldenStreetRow}. Default true.
-     */
-    splitPrefix?: boolean;
-}
-/**
- * Aggregate outcome for a whole golden version.
- */
-export interface RelabelGoldenReport {
-    files: Record<string, {
-        entries: number;
-        changed: number;
-        flagged: number;
-        prefixSplit: number;
-        counts: GoldenRelabelCounts;
-    }>;
-    deckPath: string;
-    outputDir: string;
-    totalChanged: number;
-    totalFlagged: number;
-}
-/**
- * Relabel every `.jsonl` in a golden version dir, writing a new version dir plus a review deck and a MANIFEST that
- * records the convention, the parent, and the counts. Non-JSONL siblings (README, split manifests) are copied forward
- * so the new version is self-contained; nested split dirs (`dev/`, `test/`) are relabelled recursively.
- */
-export declare function relabelGoldenDirectory(options: RelabelGoldenOptions, report?: (line: string) => void): Promise<RelabelGoldenReport>;
-/**
- * Every relabel class that means the row was left folded, for callers that want to report the residue.
- */
-export declare function isLeftFolded(rowClass: GoldenRelabelClass): boolean;
-/**
- * True when a golden dir declares the US-split convention — i.e. it is safe to grade it with an UNFOLDED scorer.
- */
-export declare function goldenDeclaresSplitStreets(dir: string): Promise<boolean>;
-//# sourceMappingURL=golden-relabel-street.d.ts.map

package/out/tools/golden-relabel-street.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"golden-relabel-street.d.ts","sourceRoot":"","sources":["../../lib/tools/golden-relabel-street.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAwDG;AAgBH;;;GAGG;AACH,MAAM,WAAW,eAAe;IAC/B,GAAG,EAAE,MAAM,CAAA;IACX,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAClC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAC3B,cAAc,GACd,8BAA8B,GAC9B,mBAAmB,GACnB,eAAe,GACf,cAAc,GACd,oBAAoB,GACpB,2BAA2B,GAC3B,iBAAiB,GACjB,WAAW,GACX,QAAQ,GACR,kBAAkB,CAAA;AAErB;;;GAGG;AACH,MAAM,WAAW,iBAAiB;IACjC,IAAI,EAAE,mBAAmB,GAAG,eAAe,GAAG,oBAAoB,CAAA;IAClE,MAAM,EAAE,MAAM,CAAA;CACd;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC;;OAEG;IACH,GAAG,EAAE,eAAe,CAAA;IACpB,OAAO,EAAE,OAAO,CAAA;IAChB,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,KAAK,EAAE,iBAAiB,EAAE,CAAA;IAC1B;;OAEG;IACH,WAAW,EAAE,OAAO,CAAA;IACpB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAkFD;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;;;;;;OAOG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;;GAGG;AACH,wBAAgB,sBAAsB,CACrC,GAAG,EAAE,eAAe,EACpB,OAAO,GAAE,uBAA4B,GACnC,mBAAmB,CAmIrB;AAID;;GAEG;AACH,MAAM,MAAM,mBAAmB,GAAG,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAA;AAEpE;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,QAAQ,EAAE,kBAAkB,CAAA;IAC5B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC9B,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAC7B,KAAK,EAAE,iBAAiB,EAAE,CAAA;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;OAEG;IACH,WAAW,CAAC,EAAE,OAAO,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,mBAAmB;IACnC,KAAK,EAAE,MAAM,CACZ,MAAM,EACN;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,mBAAmB,CAAA;KAAE,CACvG,CAAA;IACD,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,YAAY,EAAE,MAAM,CAAA;CACpB;AA0BD;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,OAAO,EAAE,oBAAoB,EAC7B,MAAM,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAkB,GAC1C,OAAO,CAAC,mBAAmB,CAAC,CAgJ9B;AAsDD;;GAEG;AACH,wBAAgB,YAAY,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO,CAElE;AAED;;GAEG;AACH,wBAAsB,0BAA0B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC,CAY9E"}