@mailwoman/corpus 10.0.0 → 10.1.0

package/out/tools/lint/{slice → recipe-output}/vocab.js

@@ -3,42 +3,24 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic slice vs the base.
+ * Base-consistency lint: a synthetic recipe output must not label a token a tag the base dominantly labels
+ * something else, or training gets conflicting gradients on the same token and the minority (the recipe output)
+ * loses. This reads a recipe output's own (token -> tag) and checks each token against the base.
  *
- *   Ported from scripts/lint-slice-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic slice must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   slice) loses. This reads a slice's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each slice token's base tag SCOPED to the
- *        country the slice uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (slice street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint slice-vocab --slice <slice.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
+ * The check is country-scoped and full-count, because a token's correct tag is country-specific: "Paris" is a
+ * locality in FR data and a street in US "Paris Ave". A cross-country aggregate mis-judges any country-specific
+ * token. A small sample is street-biased because street sources dwarf locality sources. Each
+ * recipe-output token's base tag is therefore tallied scoped to the country the output uses it in, over a large
+ * scan. Pure-numeric tokens are excluded because house_number and postcode are context-determined. An
+ * affix-split flag (recipe output street_suffix/_prefix against base "street") is expected, because the loader's
+ * affix-relabel handles it.
  */
 import { dataRootPath } from "@mailwoman/core/data-root";
 import { tryParsingJSON } from "@mailwoman/core/json";
 import { pyRound } from "@mailwoman/core/numeric";
-import { join } from "path-ts";
+import { PathBuilder } from "path-ts";
 import { Globerator } from "spliterator/node/fs";
-import { connectDuckDB } from "#utils/parquet";
+import { openDuckDB } from "#parquet/duckdb";
 /**
  * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
  */
@@ -47,9 +29,11 @@ function stripBIO(label) {
     return head === "B-" || head === "I-" ? label.slice(2) : label;
 }
 /**
- * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
- * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
- * corpora actually contain.
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit.
+ *
+ * Pure-numeric tokens (house_number and postcode) are context-determined
+ * rather than lexical vocab, so they are excluded.
+ * `\p{Nd}` covers the decimal digits these address corpora actually contain.
  */
 function isDigit(token) {
     return token.length > 0 && /^\p{Nd}+$/u.test(token);
@@ -61,9 +45,10 @@ function pct(frac) {
     return `${pyRound(frac * 100)}%`;
 }
 /**
- * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
- * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
- * print.
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one
+ * decimal (1.0 -> "1.0"), everything else is its shortest decimal (0.5 -> "0.5").
+ *
+ * Used for the `fraction` echo so the banner matches the Python print.
  */
 function formatPyFloat(n) {
     return Number.isInteger(n) ? n.toFixed(1) : String(n);
@@ -75,8 +60,10 @@ function pad(value, width) {
     return value.padEnd(width);
 }
 /**
- * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
- * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
+ * The dominant tag of a counter as (tag, total, fraction).
+ *
+ * An empty counter yields ("", 0, 0.0), and ties go to the first-inserted tag,
+ * mirroring `Counter.most_common(1)`.
  */
 function dominant(counter) {
     let total = 0;
@@ -105,9 +92,11 @@ function bump(table, key, sub) {
     counter.set(sub, (counter.get(sub) ?? 0) + 1);
 }
 /**
- * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
- * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
- * here.
+ * Read a corpus parquet into rows, projecting only tokens, labels and country.
+ *
+ * DuckDB `to_json` returns list columns as JSON text.
+ * Gazetteer builders use the same technique for nested columns.
+ * This function parses the text back to string arrays.
  */
 async function readRows(con, path) {
     const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
@@ -127,7 +116,7 @@ async function readRows(con, path) {
     return rows;
 }
 /**
- * Read just the first row's `source` value — used to group base parts for a proportional slice.
+ * Read just the first row's `source` value, used to group base parts for a proportional sample.
  */
 async function readSource(con, path) {
     const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
@@ -135,39 +124,37 @@ async function readSource(con, path) {
     return rows.length ? String(rows[0].source) : "";
 }
 /**
- * Lint a synthetic slice's (token → tag) vocabulary against the base corpus, country-scoped.
+ * Lint a synthetic recipe output's (token → tag) vocabulary against the base corpus, country-scoped.
  */
-export async function lintSliceVocab(options) {
+export async function lintRecipeVocab(options) {
     const baseVersion = options.baseVersion ?? "v0.5.0";
-    const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
+    const baseRoot = PathBuilder.from(options.baseRoot ?? dataRootPath("corpus", "versioned"));
     const threshold = options.threshold ?? 0.7;
     const minCount = options.minCount ?? 50;
     const fraction = options.fraction ?? 1;
-    const con = await connectDuckDB();
-    // 1. the slice's own (token -> dominant tag) + the COUNTRIES it uses each token in
-    const sliceRows = await readRows(con, options.slice);
-    const sliceTags = new Map();
-    const sliceCountries = new Map();
-    for (const { tokens, labels, country } of sliceRows) {
+    using con = await openDuckDB();
+    const outputRows = await readRows(con, options.recipeOutputPath);
+    const outputTags = new Map();
+    const outputCountries = new Map();
+    for (const { tokens, labels, country } of outputRows) {
         const n = Math.min(tokens.length, labels.length);
         for (let i = 0; i < n; i++) {
             const w = tokens[i];
             const l = labels[i];
             if (isDigit(w))
-                continue; // numbers are context-determined (house_number/postcode), not lexical vocab
-            bump(sliceTags, w, stripBIO(l));
-            let set = sliceCountries.get(w);
+                continue;
+            bump(outputTags, w, stripBIO(l));
+            let set = outputCountries.get(w);
             if (!set) {
                 set = new Set();
-                sliceCountries.set(w, set);
+                outputCountries.set(w, set);
             }
             set.add(country);
         }
     }
-    const sliceVocab = new Set(sliceTags.keys());
-    console.log(`slice: ${sliceRows.length} rows, ${sliceVocab.size} unique tokens`);
-    // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
-    const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
+    const outputVocab = new Set(outputTags.keys());
+    console.log(`recipe output: ${outputRows.length} rows, ${outputVocab.size} unique tokens`);
+    const trainDir = baseRoot(baseVersion, `corpus-${baseVersion}`, "train");
     let parts = (await Globerator.from("*.parquet", {
         cwd: trainDir,
         absolute: true,
@@ -186,17 +173,16 @@ export async function lintSliceVocab(options) {
             }
             list.push(p);
         }
-        const sliced = [];
+        const sampled = [];
         for (const ps of bysrc.values()) {
             const take = Math.max(2, pyRound(ps.length * fraction));
             for (const p of ps.slice(0, take)) {
-                sliced.push(p);
+                sampled.push(p);
             }
         }
-        parts = sliced;
+        parts = sampled;
     }
     console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`);
-    // 3. tally each slice token's base tag, SCOPED to the country the slice uses it in
     const baseTags = new Map();
     for (let i = 0; i < parts.length; i++) {
         const rows = await readRows(con, parts[i]);
@@ -204,7 +190,7 @@ export async function lintSliceVocab(options) {
             const n = Math.min(tokens.length, labels.length);
             for (let j = 0; j < n; j++) {
                 const w = tokens[j];
-                if (sliceVocab.has(w) && sliceCountries.get(w).has(country)) {
+                if (outputVocab.has(w) && outputCountries.get(w).has(country)) {
                     bump(baseTags, w, stripBIO(labels[j]));
                 }
             }
@@ -213,11 +199,10 @@ export async function lintSliceVocab(options) {
             console.log(`  ...${i + 1}/${parts.length} parts`);
         }
     }
-    // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
     const flagged = [];
     const affix = [];
-    for (const w of sliceVocab) {
-        const [sTag] = dominant(sliceTags.get(w));
+    for (const w of outputVocab) {
+        const [sTag] = dominant(outputTags.get(w));
         const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
         if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
             continue;
@@ -239,11 +224,11 @@ export async function lintSliceVocab(options) {
         rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
         console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
         for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
-            console.log(`  ${pad(w, 18)} slice=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
+            console.log(`  ${pad(w, 18)} output=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
         }
     }
     if (!flagged.length) {
-        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — slice base-consistent`);
+        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — recipe output base-consistent`);
     }
     return {
         errors: flagged.length,

package/out/tools/lint/recipe-output/vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.js","sourceRoot":"","sources":["../../../../lib/tools/lint/recipe-output/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAyB,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAWnE;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAyDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA+B;IACpE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAA;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,UAAU,EAAE,CAAA;IAE9B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IAChE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE7D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAChC,IAAI,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAEhC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC5B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;IAE9C,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,CAAC,MAAM,UAAU,WAAW,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAE1F,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAExE,IAAI,KAAK,GAAG,CACX,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE;QAClC,GAAG,EAAE,QAAQ;QACb,QAAQ,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC,QAAQ,EAAE,CAAA;IAEZ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;QACF,CAAC;QAED,KAAK,GAAG,OAAO,CAAA;IAChB,CAAC;IAED,OAAO,CAAC,GAAG,CACV,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAC3G,CAAA;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,MAAM,OAAO,GAAe,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC3C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAa,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEpD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAgC;QAC7C,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QACjG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,mCAAmC,CAC/H,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}

package/out/tools/lint/recipe-output.d.ts

@@ -0,0 +1,73 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Corpus linter. Compares a new recipe output (one parquet file) against pre-computed corpus statistics and
+ * flags token-label distribution outliers, label-vacuum tokens, bigram-label collisions, anti-pattern rule
+ * matches and basic sanity problems.
+ *
+ * The markdown report goes to stdout, with optional markdown and JSON sidecars. The command exits 0 when no
+ * errors are found and 1 when any are, because warnings do not refuse. A flagged recipe output should include
+ * `lint_acknowledged: true` in its manifest before training consumes it.
+ */
+/**
+ * Options for {@linkcode lintRecipeOutput}.
+ */
+export interface LintRecipeOutputOptions {
+    /**
+     * The new recipe output parquet to lint.
+     */
+    recipeOutputPath: string;
+    /**
+     * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
+     */
+    statsPath: string;
+    /**
+     * Anti-pattern rules JSON.
+     *
+     * Default: the `lint-rules.json` beside this module.
+     */
+    rulesPath?: string;
+    /**
+     * Write the markdown report here as well as stdout.
+     */
+    outMd?: string;
+    /**
+     * Write a JSON sidecar of the flags + summary here.
+     */
+    outJSON?: string;
+}
+/**
+ * One lint flag emitted by a check.
+ */
+export interface LintFlag {
+    check: string;
+    severity: "error" | "warn";
+    token?: string;
+    bigram?: string;
+    outputLabel?: string;
+    corpusLabel?: string;
+    outputCount?: number;
+    corpusCount?: number;
+    detail: string;
+    ruleID?: string;
+}
+/**
+ * Findings summary returned by {@linkcode lintRecipeOutput}.
+ */
+export interface LintRecipeOutputSummary {
+    errors: number;
+    warnings: number;
+    findings: LintFlag[];
+    /**
+     * The rendered markdown report (also printed to stdout).
+     */
+    report: string;
+}
+/**
+ * Lint a recipe output against corpus stats and the anti-pattern rules,
+ * printing the markdown report to stdout.
+ */
+export declare function lintRecipeOutput(options: LintRecipeOutputOptions, report?: (line: string) => void): Promise<LintRecipeOutputSummary>;
+//# sourceMappingURL=recipe-output.d.ts.map

package/out/tools/lint/recipe-output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"recipe-output.d.ts","sourceRoot":"","sources":["../../../lib/tools/lint/recipe-output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAgDH;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAA;IACxB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AAmFD;;GAEG;AACH,MAAM,WAAW,QAAQ;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,uBAAuB;IACvC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,QAAQ,EAAE,CAAA;IACpB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAqND;;;GAGG;AACH,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,uBAAuB,EAChC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,uBAAuB,CAAC,CAiElC"}