@mailwoman/corpus 9.1.0 → 9.3.0

package/out/tools/lint-slice-vocab.d.ts

@@ -0,0 +1,90 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
+ *
+ *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
+ *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+/**
+ * Options for {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabOptions {
+    /**
+     * The slice parquet to lint.
+     */
+    slice: string;
+    /**
+     * Base corpus version. Default `v0.5.0`.
+     */
+    baseVersion?: string;
+    /**
+     * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
+     */
+    baseRoot?: string;
+    /**
+     * Base-majority confidence floor for a contradiction. Default 0.7.
+     */
+    threshold?: number;
+    /**
+     * Minimum base support to judge a token. Default 50.
+     */
+    minCount?: number;
+    /**
+     * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
+     */
+    fraction?: number;
+}
+/**
+ * One contradiction row: token, slice tag, base tag, base fraction, base total.
+ */
+export type SliceVocabRow = [token: string, sliceTag: string, baseTag: string, baseFrac: number, baseTotal: number];
+/**
+ * Findings summary returned by {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabSummary {
+    /**
+     * Real contradictions — the command exits 1 when nonzero.
+     */
+    errors: number;
+    /**
+     * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
+     */
+    warnings: number;
+    findings: {
+        contradictions: SliceVocabRow[];
+        affixSplits: SliceVocabRow[];
+    };
+}
+/**
+ * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export declare function lintSliceVocab(options: LintSliceVocabOptions): Promise<LintSliceVocabSummary>;
+//# sourceMappingURL=lint-slice-vocab.d.ts.map

package/out/tools/lint-slice-vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-slice-vocab.d.ts","sourceRoot":"","sources":["../../lib/tools/lint-slice-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA0JH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAkJnG"}

package/out/tools/lint-slice-vocab.js

@@ -0,0 +1,267 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
+ *
+ *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
+ *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { readDirectory } from "@mailwoman/core/fs/readers";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { pyRound } from "@mailwoman/core/numeric";
+import { join } from "path-ts";
+import { connectDuckDB } from "#utils/parquet";
+/**
+ * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
+ */
+function stripBIO(label) {
+    const head = label.slice(0, 2);
+    return head === "B-" || head === "I-" ? label.slice(2) : label;
+}
+/**
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
+ * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
+ * corpora actually contain.
+ */
+function isDigit(token) {
+    return token.length > 0 && /^\p{Nd}+$/u.test(token);
+}
+/**
+ * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
+ */
+function pct(frac) {
+    return `${pyRound(frac * 100)}%`;
+}
+/**
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
+ * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
+ * print.
+ */
+function formatPyFloat(n) {
+    return Number.isInteger(n) ? n.toFixed(1) : String(n);
+}
+/**
+ * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
+ */
+function pad(value, width) {
+    return value.padEnd(width);
+}
+/**
+ * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
+ * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
+ */
+function dominant(counter) {
+    let total = 0;
+    let bestTag = "";
+    let bestCount = -1;
+    for (const [tag, count] of counter) {
+        total += count;
+        if (count > bestCount) {
+            bestCount = count;
+            bestTag = tag;
+        }
+    }
+    if (total === 0)
+        return ["", 0, 0];
+    return [bestTag, total, bestCount / total];
+}
+/**
+ * Bump a (key -> count) tally, creating the inner counter on first sight.
+ */
+function bump(table, key, sub) {
+    let counter = table.get(key);
+    if (!counter) {
+        counter = new Map();
+        table.set(key, counter);
+    }
+    counter.set(sub, (counter.get(sub) ?? 0) + 1);
+}
+/**
+ * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
+ * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
+ * here.
+ */
+async function readRows(con, path) {
+    const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
+    const raw = result.getRowObjects();
+    const rows = [];
+    for (const r of raw) {
+        const tokens = tryParsingJSON(String(r.tokens));
+        const labels = tryParsingJSON(String(r.labels));
+        if (!Array.isArray(tokens) || !Array.isArray(labels))
+            continue;
+        rows.push({
+            tokens: tokens,
+            labels: labels,
+            country: r.country == null ? null : String(r.country),
+        });
+    }
+    return rows;
+}
+/**
+ * Read just the first row's `source` value — used to group base parts for a proportional slice.
+ */
+async function readSource(con, path) {
+    const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
+    const rows = result.getRowObjects();
+    return rows.length ? String(rows[0].source) : "";
+}
+/**
+ * Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`.
+ */
+async function globParquet(dir) {
+    let names;
+    try {
+        names = await readDirectory(dir);
+    }
+    catch {
+        return [];
+    }
+    return names
+        .filter((f) => f.endsWith(".parquet"))
+        .map((f) => join(dir, f))
+        .toSorted();
+}
+/**
+ * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export async function lintSliceVocab(options) {
+    const baseVersion = options.baseVersion ?? "v0.5.0";
+    const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
+    const threshold = options.threshold ?? 0.7;
+    const minCount = options.minCount ?? 50;
+    const fraction = options.fraction ?? 1;
+    const con = await connectDuckDB();
+    // 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
+    const sliceRows = await readRows(con, options.slice);
+    const sliceTags = new Map();
+    const sliceCountries = new Map();
+    for (const { tokens, labels, country } of sliceRows) {
+        const n = Math.min(tokens.length, labels.length);
+        for (let i = 0; i < n; i++) {
+            const w = tokens[i];
+            const l = labels[i];
+            if (isDigit(w))
+                continue; // numbers are context-determined (house_number/postcode), not lexical vocab
+            bump(sliceTags, w, stripBIO(l));
+            let set = sliceCountries.get(w);
+            if (!set) {
+                set = new Set();
+                sliceCountries.set(w, set);
+            }
+            set.add(country);
+        }
+    }
+    const sliceVocab = new Set(sliceTags.keys());
+    console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`);
+    // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
+    const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
+    let parts = await globParquet(trainDir);
+    if (!parts.length) {
+        throw new Error("no base parts found");
+    }
+    if (fraction < 1) {
+        const bysrc = new Map();
+        for (const p of parts) {
+            const src = await readSource(con, p);
+            let list = bysrc.get(src);
+            if (!list) {
+                list = [];
+                bysrc.set(src, list);
+            }
+            list.push(p);
+        }
+        const sliced = [];
+        for (const ps of bysrc.values()) {
+            const take = Math.max(2, pyRound(ps.length * fraction));
+            for (const p of ps.slice(0, take)) {
+                sliced.push(p);
+            }
+        }
+        parts = sliced;
+    }
+    console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`);
+    // 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
+    const baseTags = new Map();
+    for (let i = 0; i < parts.length; i++) {
+        const rows = await readRows(con, parts[i]);
+        for (const { tokens, labels, country } of rows) {
+            const n = Math.min(tokens.length, labels.length);
+            for (let j = 0; j < n; j++) {
+                const w = tokens[j];
+                if (sliceVocab.has(w) && sliceCountries.get(w).has(country)) {
+                    bump(baseTags, w, stripBIO(labels[j]));
+                }
+            }
+        }
+        if ((i + 1) % 100 === 0) {
+            console.log(`  ...${i + 1}/${parts.length} parts`);
+        }
+    }
+    // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
+    const flagged = [];
+    const affix = [];
+    for (const w of sliceVocab) {
+        const [sTag] = dominant(sliceTags.get(w));
+        const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
+        if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
+            continue;
+        const row = [w, sTag, bTag, bFrac, bTotal];
+        if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
+            affix.push(row);
+        }
+        else {
+            flagged.push(row);
+        }
+    }
+    const sections = [
+        ["CONTRADICTION", flagged],
+        ["affix-split (EXPECTED — affix-relabel handles)", affix],
+    ];
+    for (const [label, rows] of sections) {
+        if (!rows.length)
+            continue;
+        rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
+        console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
+        for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
+            console.log(`  ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
+        }
+    }
+    if (!flagged.length) {
+        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`);
+    }
+    return {
+        errors: flagged.length,
+        warnings: affix.length,
+        findings: { contradictions: flagged, affixSplits: affix },
+    };
+}
+//# sourceMappingURL=lint-slice-vocab.js.map

package/out/tools/lint-slice-vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-slice-vocab.js","sourceRoot":"","sources":["../../lib/tools/lint-slice-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAA;AAC1D,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,OAAO,EAAE,aAAa,EAAyB,MAAM,gBAAgB,CAAA;AAWrE;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,KAAe,CAAA;IAEnB,IAAI,CAAC;QACJ,KAAK,GAAG,MAAM,aAAa,CAAC,GAAG,CAAC,CAAA;IACjC,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;SACxB,QAAQ,EAAE,CAAA;AACb,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,aAAa,EAAE,CAAA;IAEjC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9E,IAAI,KAAK,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,CAAA;IAEvC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CACV,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAC3G,CAAA;IAED,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}

package/out/tools/lint-slice.d.ts

@@ -0,0 +1,93 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
+ *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
+ *   venue templates.
+ *
+ *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
+ *
+ *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
+ *        majority label to the corpus's majority label. Flag when the corpus has a
+ *        confidently-established majority (>66%) AND the slice's majority differs AND both have
+ *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
+ *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
+ *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
+ *        we're introducing a novel association, not shifting a distribution.
+ *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
+ *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
+ *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
+ *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
+ *        mappings — flagging matches.
+ *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
+ *
+ *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
+ *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
+ *   flagged slice should require `lint_acknowledged: true` before training consumes it.
+ *
+ *   Usage: mailwoman dev lint corpus-slice\
+ *   --slice <new-slice.parquet>\
+ *   --stats <corpus-stats.json>\
+ *   [--rules <rules.json>]\
+ *   [--out-md /tmp/lint-report.md]\
+ *   [--out-json /tmp/lint-report.json]
+ */
+/**
+ * Options for {@linkcode lintCorpusSlice}.
+ */
+export interface LintCorpusSliceOptions {
+    /**
+     * The new slice parquet to lint.
+     */
+    slicePath: string;
+    /**
+     * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
+     */
+    statsPath: string;
+    /**
+     * Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module.
+     */
+    rulesPath?: string;
+    /**
+     * Write the markdown report here as well as stdout.
+     */
+    outMd?: string;
+    /**
+     * Write a JSON sidecar of the flags + summary here.
+     */
+    outJSON?: string;
+}
+/**
+ * One lint flag emitted by a check.
+ */
+export interface LintSliceFlag {
+    check: string;
+    severity: "error" | "warn";
+    token?: string;
+    bigram?: string;
+    sliceLabel?: string;
+    corpusLabel?: string;
+    sliceCount?: number;
+    corpusCount?: number;
+    detail: string;
+    ruleID?: string;
+}
+/**
+ * Findings summary returned by {@linkcode lintCorpusSlice}.
+ */
+export interface LintCorpusSliceSummary {
+    errors: number;
+    warnings: number;
+    findings: LintSliceFlag[];
+    /**
+     * The rendered markdown report (also printed to stdout).
+     */
+    report: string;
+}
+/**
+ * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
+ */
+export declare function lintCorpusSlice(options: LintCorpusSliceOptions, report?: (line: string) => void): Promise<LintCorpusSliceSummary>;
+//# sourceMappingURL=lint-slice.d.ts.map

package/out/tools/lint-slice.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lint-slice.d.ts","sourceRoot":"","sources":["../../lib/tools/lint-slice.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AA8CH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAmFD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAqND;;GAEG;AACH,wBAAsB,eAAe,CACpC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CA6DjC"}