@mailwoman/corpus 9.4.0 → 10.0.0

package/out/tools/lint/slice/index.js

@@ -0,0 +1,322 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Corpus linter. Compares a new slice against pre-computed corpus statistics and flags patterns
+ *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
+ *   venue templates.
+ *
+ *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
+ *
+ *   1. **Token-label distribution outliers.** For each token in the new slice, compare the slice's
+ *        majority label to the corpus's majority label. Flag when the corpus has a
+ *        confidently-established majority (>66%) AND the slice's majority differs AND both have
+ *        non-trivial counts (slice ≥ 50, corpus ≥ 200).
+ *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
+ *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
+ *        we're introducing a novel association, not shifting a distribution.
+ *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in slice while the
+ *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
+ *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
+ *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
+ *        mappings — flagging matches.
+ *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of slice.
+ *
+ *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
+ *   no errors, 1 if any errors (warnings don't check). Per the design, the MANIFEST entry for a
+ *   flagged slice should require `lint_acknowledged: true` before training consumes it.
+ *
+ *   Usage: mailwoman dev lint corpus-slice\
+ *   --slice <new-slice.parquet>\
+ *   --stats <corpus-stats.json>\
+ *   [--rules <rules.json>]\
+ *   [--out-md /tmp/lint-report.md]\
+ *   [--out-json /tmp/lint-report.json]
+ */
+import { readLocalJSONFile } from "@mailwoman/core/fs/readers";
+import { writeLocalFile, writeLocalJSONFile } from "@mailwoman/core/fs/writers";
+import { resolvePackagePath } from "@mailwoman/core/module/resolvers";
+import { accumulateCooccurrences, createCooccurrenceStats, SLICE_STATS_SEP as SEP, streamTokenLabelRows, } from "#utils/slice-stats";
+/**
+ * Occurrences of a forbidden label before it is reported — one or two are noise, five is a pattern.
+ */
+const FORBIDDEN_LABEL_REPORT_THRESHOLD = 5;
+/**
+ * Examples printed per finding before the list is truncated.
+ */
+const MAX_LISTED_EXAMPLES = 20;
+/**
+ * Calibrated thresholds (DeepSeek turn 9). These can be tuned over time if new failure modes surface that the current
+ * numbers miss.
+ */
+const CORPUS_CONFIDENCE_FLOOR = 0.66;
+const SLICE_MIN_COUNT = 50;
+const CORPUS_MIN_COUNT = 200;
+const VACUUM_SLICE_MIN_COUNT = 20;
+const VACUUM_CORPUS_MIN_COUNT = 100;
+const BIGRAM_MIN_COUNT = 10;
+const ALL_O_RATIO_CEILING = 0.9;
+/**
+ * Default `lint-rules.json` path — the rules ship beside this module in the source tree. tsc does not emit
+ * readFileSync'd JSON into `out/`, so the compiled tree falls back to the source-tree copy (corpus/out/src/tools/ →
+ * corpus/src/tools/). In-repo the `node` exports condition loads this module from source anyway, so the sibling URL is
+ * the common path.
+ */
+function defaultRulesPath() {
+    return resolvePackagePath("@mailwoman/corpus", "lib", "tools", "lint-rules.json");
+}
+async function statsFromSlice(rows) {
+    const co = createCooccurrenceStats();
+    const out = {
+        rowCount: 0,
+        tokens: co.tokens,
+        bigrams: co.bigrams,
+        truncatedRows: 0,
+        allORows: 0,
+    };
+    for await (const row of rows) {
+        out.rowCount++;
+        if (row.tokens.length !== row.labels.length) {
+            out.truncatedRows++;
+            continue;
+        }
+        if (row.labels.every((l) => l === "O")) {
+            out.allORows++;
+        }
+        accumulateCooccurrences(co, row.tokens, row.labels);
+    }
+    return out;
+}
+function majorityLabel(distribution) {
+    const entries = distribution instanceof Map ? [...distribution.entries()] : Object.entries(distribution);
+    let bestLabel = "";
+    let bestCount = 0;
+    let total = 0;
+    for (const [label, count] of entries) {
+        total += count;
+        if (count > bestCount) {
+            bestCount = count;
+            bestLabel = label;
+        }
+    }
+    return { label: bestLabel, count: bestCount, total, confidence: total === 0 ? 0 : bestCount / total };
+}
+function checkDistributionOutliers(slice, corpus) {
+    const flags = [];
+    for (const [token, sliceLabelMap] of slice.tokens) {
+        const corpusLabelMap = corpus.tokens[token];
+        if (!corpusLabelMap)
+            continue;
+        const sliceMaj = majorityLabel(sliceLabelMap);
+        const corpusMaj = majorityLabel(corpusLabelMap);
+        if (corpusMaj.confidence >= CORPUS_CONFIDENCE_FLOOR &&
+            sliceMaj.label !== corpusMaj.label &&
+            sliceMaj.count >= SLICE_MIN_COUNT &&
+            corpusMaj.total >= CORPUS_MIN_COUNT) {
+            flags.push({
+                check: "distribution-outlier",
+                severity: "error",
+                token,
+                sliceLabel: sliceMaj.label,
+                corpusLabel: corpusMaj.label,
+                sliceCount: sliceMaj.count,
+                corpusCount: corpusMaj.count,
+                detail: `Token "${token}": slice majority is ${sliceMaj.label} (${sliceMaj.count}/${sliceMaj.total}, ${(sliceMaj.confidence * 100).toFixed(0)}%), corpus majority is ${corpusMaj.label} (${corpusMaj.count}/${corpusMaj.total}, ${(corpusMaj.confidence * 100).toFixed(0)}%).`,
+            });
+        }
+    }
+    return flags;
+}
+function checkLabelVacuum(slice, corpus) {
+    const flags = [];
+    for (const [token, sliceLabelMap] of slice.tokens) {
+        const corpusLabelMap = corpus.tokens[token];
+        if (!corpusLabelMap)
+            continue;
+        const corpusTotal = Object.values(corpusLabelMap).reduce((a, b) => a + b, 0);
+        if (corpusTotal < VACUUM_CORPUS_MIN_COUNT)
+            continue;
+        for (const [label, sliceCount] of sliceLabelMap) {
+            if (sliceCount < VACUUM_SLICE_MIN_COUNT)
+                continue;
+            if (corpusLabelMap[label] === undefined || corpusLabelMap[label] === 0) {
+                flags.push({
+                    check: "label-vacuum",
+                    severity: "error",
+                    token,
+                    sliceLabel: label,
+                    sliceCount,
+                    corpusCount: corpusTotal,
+                    detail: `Token "${token}": slice labels it ${label} ${sliceCount} times, but the corpus (${corpusTotal} instances of this token) has ZERO instances of this label.`,
+                });
+            }
+        }
+    }
+    return flags;
+}
+function checkBigramCollisions(slice, corpus) {
+    const flags = [];
+    for (const [bigram, sliceLabelMap] of slice.bigrams) {
+        const corpusLabelMap = corpus.bigrams[bigram];
+        if (!corpusLabelMap)
+            continue;
+        const sliceMaj = majorityLabel(sliceLabelMap);
+        const corpusMaj = majorityLabel(corpusLabelMap);
+        if (sliceMaj.label !== corpusMaj.label &&
+            sliceMaj.count >= BIGRAM_MIN_COUNT &&
+            corpusMaj.count >= BIGRAM_MIN_COUNT) {
+            const renderBigram = bigram.split(SEP).join(" ");
+            const renderSliceLabel = sliceMaj.label.split(SEP).join(" → ");
+            const renderCorpusLabel = corpusMaj.label.split(SEP).join(" → ");
+            flags.push({
+                check: "bigram-collision",
+                severity: "error",
+                bigram: renderBigram,
+                sliceLabel: renderSliceLabel,
+                corpusLabel: renderCorpusLabel,
+                sliceCount: sliceMaj.count,
+                corpusCount: corpusMaj.count,
+                detail: `Bigram "${renderBigram}": slice label-bigram is [${renderSliceLabel}] (${sliceMaj.count}×), corpus label-bigram is [${renderCorpusLabel}] (${corpusMaj.count}×). Same surface text, different structural reading.`,
+            });
+        }
+    }
+    return flags;
+}
+function checkRules(slice, rulesFile) {
+    const flags = [];
+    const compiled = rulesFile.rules.map((r) => ({
+        rule: r,
+        regex: new RegExp(r.pattern, r.pattern_case_sensitive ? "" : "i"),
+    }));
+    for (const [token, labelMap] of slice.tokens) {
+        for (const { rule, regex } of compiled) {
+            if (!regex.test(token))
+                continue;
+            for (const [label, count] of labelMap) {
+                if (rule.forbidden_labels.includes(label) && count >= FORBIDDEN_LABEL_REPORT_THRESHOLD) {
+                    flags.push({
+                        check: "anti-pattern-rule",
+                        severity: rule.severity,
+                        ruleID: rule.id,
+                        token,
+                        sliceLabel: label,
+                        sliceCount: count,
+                        detail: `Token "${token}" matched rule ${rule.id} and is labeled ${label} ${count} time(s). Rule message: ${rule.message}`,
+                    });
+                }
+            }
+        }
+    }
+    return flags;
+}
+function checkSanity(slice) {
+    const flags = [];
+    if (slice.truncatedRows > 0) {
+        flags.push({
+            check: "truncated-rows",
+            severity: "error",
+            detail: `${slice.truncatedRows} row(s) have tokens.length !== labels.length. Pipeline alignment bug.`,
+        });
+    }
+    const allORatio = slice.allORows / Math.max(1, slice.rowCount);
+    if (allORatio >= ALL_O_RATIO_CEILING) {
+        flags.push({
+            check: "all-O-slice",
+            severity: "warn",
+            detail: `${slice.allORows}/${slice.rowCount} rows (${(allORatio * 100).toFixed(0)}%) are entirely O-labeled. Slice contributes no signal.`,
+        });
+    }
+    return flags;
+}
+function renderReport(opts, slice, flags) {
+    const errors = flags.filter((f) => f.severity === "error");
+    const warns = flags.filter((f) => f.severity === "warn");
+    const verdict = !errors.length ? "**PASS** ✓" : "**FLAGGED** ⚠";
+    const lines = [
+        `# Corpus Lint: ${verdict}`,
+        "",
+        `- **Slice:** \`${opts.slicePath}\``,
+        `- **Corpus stats:** \`${opts.statsPath}\``,
+        `- **Rules:** \`${opts.rulesPath}\``,
+        `- **Slice rows:** ${slice.rowCount}`,
+        `- **Unique tokens:** ${slice.tokens.size}`,
+        `- **Unique bigrams:** ${slice.bigrams.size}`,
+        "",
+        `**Errors:** ${errors.length} (checks the slice's inclusion unless MANIFEST sets \`lint_acknowledged: true\`)`,
+        `**Warnings:** ${warns.length} (advisory)`,
+        "",
+    ];
+    if (!flags.length) {
+        lines.push("No anomalies detected.");
+        return lines.join("\n");
+    }
+    const byCheck = new Map();
+    for (const f of flags) {
+        const arr = byCheck.get(f.check) ?? [];
+        arr.push(f);
+        byCheck.set(f.check, arr);
+    }
+    for (const [check, list] of byCheck) {
+        lines.push(`## ${check} (${list.length})`);
+        lines.push("");
+        // Sort by sliceCount desc — highest-volume issues first
+        list.sort((a, b) => (b.sliceCount ?? 0) - (a.sliceCount ?? 0));
+        for (const f of list.slice(0, 20)) {
+            lines.push(`- **[${f.severity.toUpperCase()}]** ${f.detail}`);
+        }
+        if (list.length > MAX_LISTED_EXAMPLES) {
+            lines.push(`- ... and ${list.length - 20} more`);
+        }
+        lines.push("");
+    }
+    return lines.join("\n");
+}
+/**
+ * Lint a slice against corpus stats + the anti-pattern rules; print the markdown report to stdout.
+ */
+export async function lintCorpusSlice(options, report) {
+    const rulesPath = options.rulesPath ?? defaultRulesPath();
+    report?.(`Reading corpus stats from ${options.statsPath}...`);
+    const corpus = await readLocalJSONFile(options.statsPath);
+    report?.(`  ${corpus.row_count} rows from ${corpus.slice_paths.length} slice(s); ${Object.keys(corpus.tokens).length} tokens, ${Object.keys(corpus.bigrams).length} bigrams`);
+    report?.(`Reading slice from ${options.slicePath}...`);
+    const slice = await statsFromSlice(streamTokenLabelRows(options.slicePath));
+    report?.(`  ${slice.rowCount} rows`);
+    report?.(`Loading rules from ${rulesPath}...`);
+    const rulesFile = await readLocalJSONFile(rulesPath);
+    report?.(`Running checks...`);
+    const flags = [
+        ...checkDistributionOutliers(slice, corpus),
+        ...checkLabelVacuum(slice, corpus),
+        ...checkBigramCollisions(slice, corpus),
+        ...checkRules(slice, rulesFile),
+        ...checkSanity(slice),
+    ];
+    const rendered = renderReport({ slicePath: options.slicePath, statsPath: options.statsPath, rulesPath }, slice, flags);
+    console.log(rendered);
+    if (options.outMd) {
+        await writeLocalFile(rendered, options.outMd);
+    }
+    if (options.outJSON) {
+        await writeLocalJSONFile({
+            slice: options.slicePath,
+            stats: options.statsPath,
+            flags,
+            summary: {
+                errors: flags.filter((f) => f.severity === "error").length,
+                warnings: flags.filter((f) => f.severity === "warn").length,
+            },
+        }, options.outJSON);
+    }
+    const errorCount = flags.filter((f) => f.severity === "error").length;
+    const warningCount = flags.filter((f) => f.severity === "warn").length;
+    if (errorCount > 0) {
+        report?.(`LINT FAILED: ${errorCount} error(s).`);
+    }
+    else {
+        report?.("LINT PASSED.");
+    }
+    return { errors: errorCount, warnings: warningCount, findings: flags, report: rendered };
+}
+//# sourceMappingURL=index.js.map

package/out/tools/lint/slice/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../lib/tools/lint/slice/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC9D,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/E,OAAO,EAAE,kBAAkB,EAAE,MAAM,kCAAkC,CAAA;AAErE,OAAO,EACN,uBAAuB,EACvB,uBAAuB,EACvB,eAAe,IAAI,GAAG,EACtB,oBAAoB,GAEpB,MAAM,oBAAoB,CAAA;AAE3B;;GAEG;AACH,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C;;GAEG;AACH,MAAM,mBAAmB,GAAG,EAAE,CAAA;AAE9B;;;GAGG;AACH,MAAM,uBAAuB,GAAG,IAAI,CAAA;AACpC,MAAM,eAAe,GAAG,EAAE,CAAA;AAC1B,MAAM,gBAAgB,GAAG,GAAG,CAAA;AAC5B,MAAM,sBAAsB,GAAG,EAAE,CAAA;AACjC,MAAM,uBAAuB,GAAG,GAAG,CAAA;AACnC,MAAM,gBAAgB,GAAG,EAAE,CAAA;AAC3B,MAAM,mBAAmB,GAAG,GAAG,CAAA;AAE/B;;;;;GAKG;AACH,SAAS,gBAAgB;IACxB,OAAO,kBAAkB,CAAC,mBAAmB,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,CAAC,CAAA;AAClF,CAAC;AAwDD,KAAK,UAAU,cAAc,CAAC,IAA6B;IAC1D,MAAM,EAAE,GAAG,uBAAuB,EAAE,CAAA;IAEpC,MAAM,GAAG,GAAe;QACvB,QAAQ,EAAE,CAAC;QACX,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,OAAO,EAAE,EAAE,CAAC,OAAO;QACnB,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;KACX,CAAA;IAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,GAAG,CAAC,QAAQ,EAAE,CAAA;QAEd,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YAC7C,GAAG,CAAC,aAAa,EAAE,CAAA;YAEnB,SAAQ;QACT,CAAC;QAED,IAAI,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,QAAQ,EAAE,CAAA;QACf,CAAC;QAED,uBAAuB,CAAC,EAAE,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;IACpD,CAAC;IAED,OAAO,GAAG,CAAA;AACX,CAAC;AAED,SAAS,aAAa,CAAC,YAA0D;IAMhF,MAAM,OAAO,GAAG,YAAY,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC,CAAA;IACxG,IAAI,SAAS,GAAG,EAAE,CAAA;IAClB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,KAAK,GAAG,CAAC,CAAA;IAEb,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACtC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,SAAS,GAAG,KAAK,CAAA;QAClB,CAAC;IACF,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,KAAK,EAAE,CAAA;AACtG,CAAC;AA+BD,SAAS,yBAAyB,CAAC,KAAiB,EAAE,MAAmB;IACxE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,SAAS,CAAC,UAAU,IAAI,uBAAuB;YAC/C,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,eAAe;YACjC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,sBAAsB;gBAC7B,QAAQ,EAAE,OAAO;gBACjB,KAAK;gBACL,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,UAAU,KAAK,wBAAwB,QAAQ,CAAC,KAAK,KAAK,QAAQ,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,CAAC,QAAQ,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,SAAS,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK,IAAI,SAAS,CAAC,KAAK,KAAK,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;aAC9Q,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAiB,EAAE,MAAmB;IAC/D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,KAAK,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACnD,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QAE3C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAE5E,IAAI,WAAW,GAAG,uBAAuB;YAAE,SAAQ;QAEnD,KAAK,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,IAAI,aAAa,EAAE,CAAC;YACjD,IAAI,UAAU,GAAG,sBAAsB;gBAAE,SAAQ;YAEjD,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,SAAS,IAAI,cAAc,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACxE,KAAK,CAAC,IAAI,CAAC;oBACV,KAAK,EAAE,cAAc;oBACrB,QAAQ,EAAE,OAAO;oBACjB,KAAK;oBACL,UAAU,EAAE,KAAK;oBACjB,UAAU;oBACV,WAAW,EAAE,WAAW;oBACxB,MAAM,EAAE,UAAU,KAAK,sBAAsB,KAAK,IAAI,UAAU,2BAA2B,WAAW,6DAA6D;iBACnK,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAiB,EAAE,MAAmB;IACpE,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QACrD,MAAM,cAAc,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QAE7C,IAAI,CAAC,cAAc;YAAE,SAAQ;QAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,aAAa,CAAC,CAAA;QAC7C,MAAM,SAAS,GAAG,aAAa,CAAC,cAAc,CAAC,CAAA;QAE/C,IACC,QAAQ,CAAC,KAAK,KAAK,SAAS,CAAC,KAAK;YAClC,QAAQ,CAAC,KAAK,IAAI,gBAAgB;YAClC,SAAS,CAAC,KAAK,IAAI,gBAAgB,EAClC,CAAC;YACF,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAChD,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC9D,MAAM,iBAAiB,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAEhE,KAAK,CAAC,IAAI,CAAC;gBACV,KAAK,EAAE,kBAAkB;gBACzB,QAAQ,EAAE,OAAO;gBACjB,MAAM,EAAE,YAAY;gBACpB,UAAU,EAAE,gBAAgB;gBAC5B,WAAW,EAAE,iBAAiB;gBAC9B,UAAU,EAAE,QAAQ,CAAC,KAAK;gBAC1B,WAAW,EAAE,SAAS,CAAC,KAAK;gBAC5B,MAAM,EAAE,WAAW,YAAY,6BAA6B,gBAAgB,MAAM,QAAQ,CAAC,KAAK,+BAA+B,iBAAiB,MAAM,SAAS,CAAC,KAAK,sDAAsD;aAC3N,CAAC,CAAA;QACH,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,UAAU,CAAC,KAAiB,EAAE,SAAwB;IAC9D,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,EAAE,CAAC;QACP,KAAK,EAAE,IAAI,MAAM,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;KACjE,CAAC,CAAC,CAAA;IAEH,KAAK,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9C,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACxC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC;gBAAE,SAAQ;YAEhC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,gCAAgC,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC;wBACV,KAAK,EAAE,mBAAmB;wBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;wBACvB,MAAM,EAAE,IAAI,CAAC,EAAE;wBACf,KAAK;wBACL,UAAU,EAAE,KAAK;wBACjB,UAAU,EAAE,KAAK;wBACjB,MAAM,EAAE,UAAU,KAAK,kBAAkB,IAAI,CAAC,EAAE,mBAAmB,KAAK,IAAI,KAAK,2BAA2B,IAAI,CAAC,OAAO,EAAE;qBAC1H,CAAC,CAAA;gBACH,CAAC;YACF,CAAC;QACF,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,WAAW,CAAC,KAAiB;IACrC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,IAAI,KAAK,CAAC,aAAa,GAAG,CAAC,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,gBAAgB;YACvB,QAAQ,EAAE,OAAO;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,aAAa,uEAAuE;SACrG,CAAC,CAAA;IACH,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE9D,IAAI,SAAS,IAAI,mBAAmB,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC;YACV,KAAK,EAAE,aAAa;YACpB,QAAQ,EAAE,MAAM;YAChB,MAAM,EAAE,GAAG,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,UAAU,CAAC,SAAS,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,yDAAyD;SAC1I,CAAC,CAAA;IACH,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CACpB,IAAiE,EACjE,KAAiB,EACjB,KAAsB;IAEtB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAA;IAC1D,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAA;IACxD,MAAM,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,eAAe,CAAA;IAE/D,MAAM,KAAK,GAAa;QACvB,kBAAkB,OAAO,EAAE;QAC3B,EAAE;QACF,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,yBAAyB,IAAI,CAAC,SAAS,IAAI;QAC3C,kBAAkB,IAAI,CAAC,SAAS,IAAI;QACpC,qBAAqB,KAAK,CAAC,QAAQ,EAAE;QACrC,wBAAwB,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE;QAC3C,yBAAyB,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE;QAC7C,EAAE;QACF,eAAe,MAAM,CAAC,MAAM,kFAAkF;QAC9G,iBAAiB,KAAK,CAAC,MAAM,aAAa;QAC1C,EAAE;KACF,CAAA;IAED,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAA;QAEpC,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACxB,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAA2B,CAAA;IAElD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QACtC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACX,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,IAAI,CAAC,MAAM,GAAG,CAAC,CAAA;QAC1C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACd,wDAAwD;QACxD,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,CAAC,CAAA;QAE9D,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAA;QAC9D,CAAC;QAED,IAAI,IAAI,CAAC,MAAM,GAAG,mBAAmB,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,GAAG,EAAE,OAAO,CAAC,CAAA;QACjD,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACf,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAA+B,EAC/B,MAA+B;IAE/B,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,gBAAgB,EAAE,CAAA;IACzD,MAAM,EAAE,CAAC,6BAA6B,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAC7D,MAAM,MAAM,GAAG,MAAM,iBAAiB,CAAc,OAAO,CAAC,SAAS,CAAC,CAAA;IAEtE,MAAM,EAAE,CACP,KAAK,MAAM,CAAC,SAAS,cAAc,MAAM,CAAC,WAAW,CAAC,MAAM,cAAc,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,YAAY,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,UAAU,CACnK,CAAA;IAED,MAAM,EAAE,CAAC,sBAAsB,OAAO,CAAC,SAAS,KAAK,CAAC,CAAA;IAEtD,MAAM,KAAK,GAAG,MAAM,cAAc,CAAC,oBAAoB,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAA;IAE3E,MAAM,EAAE,CAAC,KAAK,KAAK,CAAC,QAAQ,OAAO,CAAC,CAAA;IAEpC,MAAM,EAAE,CAAC,sBAAsB,SAAS,KAAK,CAAC,CAAA;IAC9C,MAAM,SAAS,GAAG,MAAM,iBAAiB,CAAgB,SAAS,CAAC,CAAA;IAEnE,MAAM,EAAE,CAAC,mBAAmB,CAAC,CAAA;IAE7B,MAAM,KAAK,GAAoB;QAC9B,GAAG,yBAAyB,CAAC,KAAK,EAAE,MAAM,CAAC;QAC3C,GAAG,gBAAgB,CAAC,KAAK,EAAE,MAAM,CAAC;QAClC,GAAG,qBAAqB,CAAC,KAAK,EAAE,MAAM,CAAC;QACvC,GAAG,UAAU,CAAC,KAAK,EAAE,SAAS,CAAC;QAC/B,GAAG,WAAW,CAAC,KAAK,CAAC;KACrB,CAAA;IAED,MAAM,QAAQ,GAAG,YAAY,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,KAAK,CAAC,CAAA;IAEtH,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAA;IAErB,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QACnB,MAAM,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;IAED,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QACrB,MAAM,kBAAkB,CACvB;YACC,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK,EAAE,OAAO,CAAC,SAAS;YACxB,KAAK;YACL,OAAO,EAAE;gBACR,MAAM,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM;gBAC1D,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM;aAC3D;SACD,EACD,OAAO,CAAC,OAAO,CACf,CAAA;IACF,CAAC;IAED,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IACrE,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IAEtE,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,CAAC,gBAAgB,UAAU,YAAY,CAAC,CAAA;IACjD,CAAC;SAAM,CAAC;QACP,MAAM,EAAE,CAAC,cAAc,CAAC,CAAA;IACzB,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAA;AACzF,CAAC"}

package/out/tools/lint/slice/vocab.d.ts

@@ -0,0 +1,90 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
+ *
+ *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
+ *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
+ *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
+ *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
+ *   the Python default.
+ *
+ *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
+ *   something else, or training gets conflicting gradients on the same token and the minority (the
+ *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
+ *
+ *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
+ *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
+ *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
+ *
+ *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
+ *        retry both false-flagged FR cities as "street" from US street-contexts).
+ *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
+ *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
+ *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
+ *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
+ *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
+ *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
+ *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
+ *
+ *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
+ *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
+ *   50]
+ */
+/**
+ * Options for {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabOptions {
+    /**
+     * The slice parquet to lint.
+     */
+    slice: string;
+    /**
+     * Base corpus version. Default `v0.5.0`.
+     */
+    baseVersion?: string;
+    /**
+     * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
+     */
+    baseRoot?: string;
+    /**
+     * Base-majority confidence floor for a contradiction. Default 0.7.
+     */
+    threshold?: number;
+    /**
+     * Minimum base support to judge a token. Default 50.
+     */
+    minCount?: number;
+    /**
+     * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
+     */
+    fraction?: number;
+}
+/**
+ * One contradiction row: token, slice tag, base tag, base fraction, base total.
+ */
+export type SliceVocabRow = [token: string, sliceTag: string, baseTag: string, baseFrac: number, baseTotal: number];
+/**
+ * Findings summary returned by {@linkcode lintSliceVocab}.
+ */
+export interface LintSliceVocabSummary {
+    /**
+     * Real contradictions — the command exits 1 when nonzero.
+     */
+    errors: number;
+    /**
+     * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
+     */
+    warnings: number;
+    findings: {
+        contradictions: SliceVocabRow[];
+        affixSplits: SliceVocabRow[];
+    };
+}
+/**
+ * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export declare function lintSliceVocab(options: LintSliceVocabOptions): Promise<LintSliceVocabSummary>;
+//# sourceMappingURL=vocab.d.ts.map

package/out/tools/lint/slice/vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.d.ts","sourceRoot":"","sources":["../../../../lib/tools/lint/slice/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAwIH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAwJnG"}