@mailwoman/corpus 9.2.0 → 9.3.0

package/out/src/tools/lint-shard-vocab.js

@@ -1,281 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
- *
- *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
- *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
- */
-import { readdirSync } from "node:fs";
-import { join } from "node:path";
-import { tryParsingJSON } from "@mailwoman/core/objects";
-import { dataRootPath } from "@mailwoman/core/utils";
-/**
- * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
- */
-function stripBIO(label) {
-    const head = label.slice(0, 2);
-    return head === "B-" || head === "I-" ? label.slice(2) : label;
-}
-/**
- * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
- * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
- * corpora actually contain.
- */
-function isDigit(token) {
-    return token.length > 0 && /^\p{Nd}+$/u.test(token);
-}
-/**
- * Round half to even (banker's rounding) — Python's built-in `round()` and `format(..., ".0%")` both use it, so percent
- * strings and the proportional `fraction` slice match the Python output exactly.
- */
-function pyRound(x) {
-    const floor = Math.floor(x);
-    const diff = x - floor;
-    if (diff < 0.5)
-        return floor;
-    if (diff > 0.5)
-        return floor + 1;
-    return floor % 2 === 0 ? floor : floor + 1;
-}
-/**
- * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
- */
-function pct(frac) {
-    return `${pyRound(frac * 100)}%`;
-}
-/**
- * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
- * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
- * print.
- */
-function pyFloat(n) {
-    return Number.isInteger(n) ? n.toFixed(1) : String(n);
-}
-/**
- * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
- */
-function pad(value, width) {
-    return value.padEnd(width);
-}
-/**
- * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
- * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
- */
-function dominant(counter) {
-    let total = 0;
-    let bestTag = "";
-    let bestCount = -1;
-    for (const [tag, count] of counter) {
-        total += count;
-        if (count > bestCount) {
-            bestCount = count;
-            bestTag = tag;
-        }
-    }
-    if (total === 0)
-        return ["", 0, 0];
-    return [bestTag, total, bestCount / total];
-}
-/**
- * Bump a (key -> count) tally, creating the inner counter on first sight.
- */
-function bump(table, key, sub) {
-    let counter = table.get(key);
-    if (!counter) {
-        counter = new Map();
-        table.set(key, counter);
-    }
-    counter.set(sub, (counter.get(sub) ?? 0) + 1);
-}
-/**
- * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
- * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
- * here.
- */
-async function readRows(con, path) {
-    const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
-    const raw = result.getRowObjects();
-    const rows = [];
-    for (const r of raw) {
-        const tokens = tryParsingJSON(String(r.tokens));
-        const labels = tryParsingJSON(String(r.labels));
-        if (!Array.isArray(tokens) || !Array.isArray(labels))
-            continue;
-        rows.push({
-            tokens: tokens,
-            labels: labels,
-            country: r.country == null ? null : String(r.country),
-        });
-    }
-    return rows;
-}
-/**
- * Read just the first row's `source` value — used to group base parts for a proportional slice.
- */
-async function readSource(con, path) {
-    const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
-    const rows = result.getRowObjects();
-    return rows.length ? String(rows[0].source) : "";
-}
-/**
- * Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`.
- */
-function globParquet(dir) {
-    let names;
-    try {
-        names = readdirSync(dir);
-    }
-    catch {
-        return [];
-    }
-    return names
-        .filter((f) => f.endsWith(".parquet"))
-        .map((f) => join(dir, f))
-        .toSorted();
-}
-/**
- * Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
- */
-export async function lintShardVocab(options) {
-    const baseVersion = options.baseVersion ?? "v0.5.0";
-    const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
-    const threshold = options.threshold ?? 0.7;
-    const minCount = options.minCount ?? 50;
-    const fraction = options.fraction ?? 1;
-    // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const instance = await DuckDBInstance.create();
-    const con = await instance.connect();
-    // 1. the shard's own (token -> dominant tag) + the COUNTRIES it uses each token in
-    const shardRows = await readRows(con, options.shard);
-    const shardTags = new Map();
-    const shardCountries = new Map();
-    for (const { tokens, labels, country } of shardRows) {
-        const n = Math.min(tokens.length, labels.length);
-        for (let i = 0; i < n; i++) {
-            const w = tokens[i];
-            const l = labels[i];
-            if (isDigit(w))
-                continue; // numbers are context-determined (house_number/postcode), not lexical vocab
-            bump(shardTags, w, stripBIO(l));
-            let set = shardCountries.get(w);
-            if (!set) {
-                set = new Set();
-                shardCountries.set(w, set);
-            }
-            set.add(country);
-        }
-    }
-    const shardVocab = new Set(shardTags.keys());
-    console.log(`shard: ${shardRows.length} rows, ${shardVocab.size} unique tokens`);
-    // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
-    const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
-    let parts = globParquet(trainDir);
-    if (!parts.length) {
-        throw new Error("no base parts found");
-    }
-    if (fraction < 1) {
-        const bysrc = new Map();
-        for (const p of parts) {
-            const src = await readSource(con, p);
-            let list = bysrc.get(src);
-            if (!list) {
-                list = [];
-                bysrc.set(src, list);
-            }
-            list.push(p);
-        }
-        const sliced = [];
-        for (const ps of bysrc.values()) {
-            const take = Math.max(2, pyRound(ps.length * fraction));
-            for (const p of ps.slice(0, take)) {
-                sliced.push(p);
-            }
-        }
-        parts = sliced;
-    }
-    console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${pyFloat(fraction)}), COUNTRY-scoped`);
-    // 3. tally each shard token's base tag, SCOPED to the country the shard uses it in
-    const baseTags = new Map();
-    for (let i = 0; i < parts.length; i++) {
-        const rows = await readRows(con, parts[i]);
-        for (const { tokens, labels, country } of rows) {
-            const n = Math.min(tokens.length, labels.length);
-            for (let j = 0; j < n; j++) {
-                const w = tokens[j];
-                if (shardVocab.has(w) && shardCountries.get(w).has(country)) {
-                    bump(baseTags, w, stripBIO(labels[j]));
-                }
-            }
-        }
-        if ((i + 1) % 100 === 0) {
-            console.log(`  ...${i + 1}/${parts.length} parts`);
-        }
-    }
-    // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
-    const flagged = [];
-    const affix = [];
-    for (const w of shardVocab) {
-        const [sTag] = dominant(shardTags.get(w));
-        const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
-        if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
-            continue;
-        const row = [w, sTag, bTag, bFrac, bTotal];
-        if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
-            affix.push(row);
-        }
-        else {
-            flagged.push(row);
-        }
-    }
-    const sections = [
-        ["CONTRADICTION", flagged],
-        ["affix-split (EXPECTED — affix-relabel handles)", affix],
-    ];
-    for (const [label, rows] of sections) {
-        if (!rows.length)
-            continue;
-        rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
-        console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
-        for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
-            console.log(`  ${pad(w, 18)} shard=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
-        }
-    }
-    if (!flagged.length) {
-        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — shard base-consistent`);
-    }
-    return {
-        errors: flagged.length,
-        warnings: affix.length,
-        findings: { contradictions: flagged, affixSplits: affix },
-    };
-}
-//# sourceMappingURL=lint-shard-vocab.js.map

package/out/src/tools/lint-shard-vocab.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard-vocab.js","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAWpD;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAA;IAEtB,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAE5B,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,GAAG,CAAC,CAAA;IAEhC,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AASD;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC/B,IAAI,KAAe,CAAA;IAEnB,IAAI,CAAC;QACJ,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;SACxB,QAAQ,EAAE,CAAA;AACb,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,gFAAgF;IAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IAEpC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9E,IAAI,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IAElH,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}

package/out/src/tools/lint-shard.d.ts

@@ -1,93 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Corpus linter. Compares a new shard against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
- *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new shard, compare the shard's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the shard's majority differs AND both have
- *        non-trivial counts (shard ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in shard while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of shard.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't gate). Per the design, the MANIFEST entry for a
- *   flagged shard should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-shard\
- *   --shard <new-shard.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
- */
-/**
- * Options for {@linkcode lintCorpusShard}.
- */
-export interface LintCorpusShardOptions {
-    /**
-     * The new shard parquet to lint.
-     */
-    shardPath: string;
-    /**
-     * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
-     */
-    statsPath: string;
-    /**
-     * Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module.
-     */
-    rulesPath?: string;
-    /**
-     * Write the markdown report here as well as stdout.
-     */
-    outMd?: string;
-    /**
-     * Write a JSON sidecar of the flags + summary here.
-     */
-    outJSON?: string;
-}
-/**
- * One lint flag emitted by a check.
- */
-export interface LintShardFlag {
-    check: string;
-    severity: "error" | "warn";
-    token?: string;
-    bigram?: string;
-    shardLabel?: string;
-    corpusLabel?: string;
-    shardCount?: number;
-    corpusCount?: number;
-    detail: string;
-    ruleID?: string;
-}
-/**
- * Findings summary returned by {@linkcode lintCorpusShard}.
- */
-export interface LintCorpusShardSummary {
-    errors: number;
-    warnings: number;
-    findings: LintShardFlag[];
-    /**
-     * The rendered markdown report (also printed to stdout).
-     */
-    report: string;
-}
-/**
- * Lint a shard against corpus stats + the anti-pattern rules; print the markdown report to stdout.
- */
-export declare function lintCorpusShard(options: LintCorpusShardOptions, report?: (line: string) => void): Promise<LintCorpusShardSummary>;
-//# sourceMappingURL=lint-shard.d.ts.map

package/out/src/tools/lint-shard.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AA+CH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AA6HD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAqND;;GAEG;AACH,wBAAsB,eAAe,CACpC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAiEjC"}