@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/audit.js

@@ -3,113 +3,98 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   `mailwoman corpus audit` — per-source slice-count vs source_weight diagnostic.
+ *   `mailwoman corpus audit` — per-source parquet-file count vs source_weight diagnostic.
  *
- *   Reads a corpus dir's MANIFEST.json (or scans slices directly), counts slices per source,
- *   optionally loads a training config to pair the counts with the configured source_weights, and
- *   reports the estimated sampled-row distribution at training time.
+ *   Reads a corpus dir's manifest.json (or scans the parquet files directly), counts files per
+ *   source, optionally loads a training config to pair the counts with the configured
+ *   source_weights. It reports the estimated sampled-row distribution at training time.
  *
- *   Would have caught v0.3.0's "NAD = 411/674 train slices × 2.0 weight = ~75% of sampled mix"
+ *   Would have caught v0.3.0's "NAD = 411/674 train files × 2.0 weight = ~75% of sampled mix"
  *   finding before the v0.3.0 retrospective surfaced it.
  *
- *   Emits warnings to stderr and the audit table to stdout; never throws on an empty corpus.
+ *   Emits warnings to stderr and the audit table to stdout. never throws on an empty corpus.
  */
-import { pathExists, readDirectory, readLocalBuffer, readLocalJSONFile } from "@mailwoman/core/fs/readers";
-import { basename, join } from "path-ts";
-import { TextSpliterator } from "spliterator";
+import { pathExists, readLocalJSONFile, readLocalTextFile } from "@mailwoman/core/fs/readers";
+import { basename, resolvePathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { readConfigView } from "#source-register/effective-manifest";
 /**
- * Share of a slice one source may hold before the mix is flagged as dominated by it.
+ * Share of the sampled mix one source may hold before the mix is flagged as dominated by it.
  */
 const DOMINANT_SOURCE_SHARE = 0.4;
 /**
- * How far the leading source may outweigh the runner-up before the mix is flagged. Catches the case where no single
- * source clears {@link DOMINANT_SOURCE_SHARE} but the distribution is still lopsided.
+ * How far the leading source may outweigh the runner-up before the mix is flagged.
+ *
+ * Catches the case where no single source clears {@link DOMINANT_SOURCE_SHARE}
+ * but the distribution is still lopsided.
  */
 const MAX_TOP_TO_RUNNER_UP_RATIO = 1.5;
 /**
- * Try parsing a training YAML's source_weights as a minimal regex-based extract. We don't pull in a YAML lib for this
- * script — the syntax is so small that a regex over the source_weights block is sufficient + keeps the script
- * dep-free.
+ * Read a training config's `source_weights`, or `null` when the file is absent.
+ *
+ * The scan lives in `readConfigView`, which the effective-manifest reader also calls,
+ * so one reading of the hand-written config subset serves both.
+ * This wrapper keeps the absent-file case a `null` rather than a throw, because
+ * `corpus audit` reports a corpus without a config rather than refusing to audit it.
  */
 async function parseConfig(configPath) {
     if (!(await pathExists(configPath)))
         return null;
-    const weights = {};
-    let inBlock = false;
-    let blockIndent = -1;
-    for (const raw of TextSpliterator.from(await readLocalBuffer(configPath))) {
-        const sourceWeightsMatch = raw.match(/^([\t ]*)source_weights:\s*$/);
-        if (sourceWeightsMatch) {
-            inBlock = true;
-            blockIndent = sourceWeightsMatch[1].length;
-            continue;
-        }
-        if (!inBlock)
-            continue;
-        // Skip blank lines and comments.
-        if (/^[\t ]*(#|$)/.test(raw))
-            continue;
-        // Lines indented MORE than `source_weights:` are entries; lines with ≤ indent end the block.
-        const indent = raw.match(/^[\t ]*/)[0].length;
-        if (indent <= blockIndent) {
-            inBlock = false;
-            continue;
-        }
-        const m = raw.match(/^[\t ]+([\w-]+):\s*([\d.]+)/);
-        if (m) {
-            weights[m[1]] = Number.parseFloat(m[2]);
-        }
-    }
-    return { sourceWeights: weights };
+    return { sourceWeights: readConfigView(await readLocalTextFile(configPath)).sourceWeights };
 }
 /**
- * Scan a corpus directory's slices (typically under <corpus_dir>/train, /val, /test) and count slices per source per
- * split.
+ * Scan a corpus directory's parquet files (typically under <corpus_dir>/train, /val, /test)
+ * and count files per source per split.
  */
-async function scanSlices(corpusDir, sampleCount) {
-    const stats = { bySplit: {}, totalSlices: 0, totalFiles: 0 };
+async function scanParquetFiles(corpusDir, sampleCount) {
+    const stats = { bySplit: {}, totalCounted: 0, totalFiles: 0 };
     for (const split of ["train", "val", "test"]) {
-        const splitDir = join(corpusDir, split);
+        const splitDir = resolvePathBuilder(corpusDir, split);
         if (!(await pathExists(splitDir)))
             continue;
-        const files = (await readDirectory(splitDir)).filter((f) => f.endsWith(".parquet")).toSorted();
+        const files = await Globerator.files("parquet", { cwd: splitDir, absolute: false, recursive: false }).toSorted();
         stats.totalFiles += files.length;
         const sampleEvery = Math.max(1, Math.floor(files.length / sampleCount));
         const sampled = files.filter((_, i) => i % sampleEvery === 0).slice(0, sampleCount);
         const splitMap = {};
         // We can't read parquet without a dep, so we infer source from filenames where possible.
-        // The corpus build typically writes deterministically by source — fall back to "<unknown>"
-        // when filename gives no hint. For accurate per-source counts on real corpora, the
-        // MANIFEST.json route below is preferred.
+        // The corpus build typically writes deterministically by source —
+        // fall back to "<unknown>" when filename gives no hint.
+        // For accurate per-source counts on real corpora, the manifest.json route below is preferred.
         for (const f of sampled) {
             const inferred = inferSourceFromFilename(f);
             splitMap[inferred] = (splitMap[inferred] ?? 0) + 1;
         }
-        // Scale to estimated full-slice counts.
+        // Scale to estimated full-file counts.
         const scale = files.length / Math.max(sampled.length, 1);
         for (const k of Object.keys(splitMap)) {
             splitMap[k] = Math.round(splitMap[k] * scale);
         }
         stats.bySplit[split] = splitMap;
-        stats.totalSlices += files.length;
+        stats.totalCounted += files.length;
     }
     return stats;
 }
 function inferSourceFromFilename(filename) {
-    // Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet. The latter (current
-    // build at corpus-v0.3.0) gives no source signal in the filename — see manifestScan() for the
-    // authoritative path. Return "<unknown>" so the caller flags this case.
+    // Many corpus builds write part-<source>-<n>.parquet or part-<n>.parquet.
+    // The latter (current build at corpus-v0.3.0) gives no source signal in the filename.
+    // See manifestScan() for the authoritative path.
+    // Return "<unknown>" so the caller flags this case.
     const m = basename(filename).match(/part-([\w-]+)-\d+\.parquet$/);
     if (m && m[1] !== undefined)
         return m[1];
     return "<unknown>";
 }
 /**
- * Known source name prefixes. Corpus-v0.3.0 uses these as `source_id` prefixes; matching against the longest prefix
- * that fits a given `first_source_id` recovers the canonical source name.
+ * Known source name prefixes.
+ *
+ * Corpus-v0.3.0 uses these as `source_id` prefixes.
+ * The parser matches the longest prefix that fits a given `first_source_id`
+ * recovers the canonical source name.
  *
- * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches `usgov-nad` rather than `usgov`. Sorted
- * descending by length at use site.
+ * Order matters: longer prefixes must be tried first so `usgov-nad-...` matches
+ * `usgov-nad` rather than `usgov`.
+ * Sorted descending by length at use site.
  */
 const KNOWN_SOURCE_PREFIXES = [
     "wof-admin",
@@ -145,61 +130,67 @@ function sourceFromID(sourceID, knownPrefixes) {
     return "<unknown>";
 }
 /**
- * Prefer reading MANIFEST.json when present — uses each slice's `first_source_id` + prefix matching to recover the
- * source name. Falls back to scanSlices when MANIFEST is absent.
+ * Prefer reading manifest.json when present — uses each file's `first_source_id` +
+ * prefix matching to recover the source name.
+ *
+ * Falls back to scanParquetFiles when manifest is absent.
+ *
+ * Note: corpus-v0.3.0 files can mix sources (see `last_source_id` differing from `first_source_id`).
+ * The first-row source is an approximation.
+ *
+ * A full read of the parquet source column would be authoritative but requires a parquet dependency.
  *
- * NOTE: corpus-v0.3.0 slices can mix sources (see `last_source_id` differing from `first_source_id`). The first-row
- * source is an approximation; reading the parquet's full source column would be authoritative but requires a parquet
- * dep. For audit purposes the first-row approximation is accurate within ~5% for the corpus-v0.3.0 shape (most slices
- * are >95% one source).
+ * For audit purposes the first-row approximation is accurate within ~5% for the
+ * corpus-v0.3.0 shape (most files are >95% one source).
  */
 async function manifestScan(corpusDir, knownPrefixes) {
-    const manifestPath = join(corpusDir, "MANIFEST.json");
+    const manifestPath = resolvePathBuilder(corpusDir, "MANIFEST.json");
     if (!(await pathExists(manifestPath)))
         return null;
     const manifest = await readLocalJSONFile(manifestPath);
     if (!Array.isArray(manifest.slices))
         return null;
     const bySplit = {};
-    for (const slice of manifest.slices) {
-        const split = slice.split;
-        const src = slice.source ?? sourceFromID(slice.first_source_id ?? "", knownPrefixes);
+    for (const file of manifest.slices) {
+        const split = file.split;
+        const src = file.source ?? sourceFromID(file.first_source_id ?? "", knownPrefixes);
         bySplit[split] ??= {};
         bySplit[split][src] = (bySplit[split][src] ?? 0) + 1;
     }
     const total = Object.values(bySplit).reduce((sum, m) => sum + Object.values(m).reduce((a, b) => a + b, 0), 0);
-    return { bySplit, totalSlices: total, totalFiles: total };
+    return { bySplit, totalCounted: total, totalFiles: total };
 }
 function buildAuditRows(stats, weights) {
-    const totalSlices = Object.values(stats).reduce((a, b) => a + b, 0);
+    const totalCounted = Object.values(stats).reduce((a, b) => a + b, 0);
     const allSources = new Set([...Object.keys(stats), ...Object.keys(weights)]);
     const rows = [];
-    // Compute effective sample weight: slice_count × source_weight. Sources with no weight get the
-    // "—" marker (loader skips them).
+    // Compute effective sample weight: file_count × source_weight.
+    // Sources with no weight get the "—" marker (loader skips them).
     const sampleWeights = [];
     for (const src of allSources) {
-        const slices = stats[src] ?? 0;
+        const files = stats[src] ?? 0;
         const weight = weights[src];
-        const effective = weight !== undefined ? slices * weight : 0;
+        const effective = weight !== undefined ? files * weight : 0;
         sampleWeights.push([src, effective]);
     }
     const totalSampleWeight = sampleWeights.reduce((a, [, w]) => a + w, 0);
     for (const src of allSources) {
-        const slices = stats[src] ?? 0;
+        const files = stats[src] ?? 0;
         const weight = weights[src] ?? "—";
-        const effective = typeof weight === "number" ? (slices * weight) / Math.max(totalSampleWeight, 1) : "—";
+        const effective = typeof weight === "number" ? (files * weight) / Math.max(totalSampleWeight, 1) : "—";
         rows.push({
             source: src,
-            slices,
-            slicePct: totalSlices > 0 ? slices / totalSlices : 0,
+            files,
+            filePct: totalCounted > 0 ? files / totalCounted : 0,
             weight,
             effectiveSamplePct: typeof effective === "number" ? effective : "—",
         });
     }
     // Flag the dominator: empirically calibrated against the v0.3.0 → v0.4.0 retrospective.
-    // v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space dilution
-    // was responsible for the coarse-F1 regression. So flag a source as "concentration warning"
-    // when it's above 40% effective sample OR more than 1.5× the next-highest.
+    // v0.3.0 had usgov-nad at 52% effective sample (1.9× ban); the resulting label-space
+    // dilution was responsible for the coarse-F1 regression.
+    // So flag a source as "concentration warning" when it's above 40% effective sample
+    // or more than 1.5× the next-highest.
     const numeric = rows.filter((r) => typeof r.effectiveSamplePct === "number");
     numeric.sort((a, b) => b.effectiveSamplePct - a.effectiveSamplePct);
     if (numeric.length) {
@@ -210,7 +201,7 @@ function buildAuditRows(stats, weights) {
             top.overweightFactor = next > 0 ? top.effectiveSamplePct / next : Infinity;
         }
     }
-    rows.sort((a, b) => b.slices - a.slices);
+    rows.sort((a, b) => b.files - a.files);
     return rows;
 }
 function formatPct(v) {
@@ -223,14 +214,14 @@ function printReport(corpusDir, configPath, stats, rows) {
     if (configPath) {
         console.log(`Config:        ${configPath}`);
     }
-    console.log(`Total slices:  ${stats.totalSlices}${stats.totalFiles !== stats.totalSlices ? ` (${stats.totalFiles} files on disk)` : ""}`);
+    console.log(`Total files:   ${stats.totalCounted}${stats.totalFiles !== stats.totalCounted ? ` (${stats.totalFiles} files on disk)` : ""}`);
     console.log("");
     const trainStats = stats.bySplit["train"];
     if (trainStats) {
         const total = Object.values(trainStats).reduce((a, b) => a + b, 0);
-        console.log(`Train split: ${total} slices`);
+        console.log(`Train split: ${total} files`);
         console.log("");
-        const headers = ["source", "slices", "slice %", "weight", "eff. sample %"];
+        const headers = ["source", "files", "file %", "weight", "eff. sample %"];
         const widths = [22, 8, 10, 8, 14];
         const fmtRow = (cells) => cells.map((c, i) => c.padEnd(widths[i])).join("  ");
         console.log(fmtRow(headers));
@@ -238,8 +229,8 @@ function printReport(corpusDir, configPath, stats, rows) {
         for (const row of rows) {
             console.log(fmtRow([
                 row.source,
-                String(row.slices),
-                formatPct(row.slicePct),
+                String(row.files),
+                formatPct(row.filePct),
                 typeof row.weight === "number" ? row.weight.toFixed(2) : "—",
                 formatPct(row.effectiveSamplePct),
             ]));
@@ -257,24 +248,25 @@ function printReport(corpusDir, configPath, stats, rows) {
         else {
             console.log("✓ No single-source concentration (top source < 40% effective sample AND < 1.5× next).");
         }
-        const missingWeights = rows.filter((r) => r.weight === "—" && r.slices > 0);
+        const missingWeights = rows.filter((r) => r.weight === "—" && r.files > 0);
         if (missingWeights.length && configPath) {
             console.error(`⚠ Sources present in corpus but absent from config.source_weights ` +
                 `(loader will skip them): ${missingWeights.map((r) => r.source).join(", ")}`);
         }
-        const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.slices === 0);
+        const orphanWeights = rows.filter((r) => typeof r.weight === "number" && r.files === 0);
         if (orphanWeights.length) {
-            console.error(`⚠ Sources weighted in config but no slices found in corpus ` +
+            console.error(`⚠ Sources weighted in config but no parquet files found in corpus ` +
                 `(no-op weights): ${orphanWeights.map((r) => r.source).join(", ")}`);
         }
     }
 }
 export async function audit(opts) {
     const config = opts.configPath ? await parseConfig(opts.configPath) : null;
-    // Compose the known-prefix list from both the hardcoded set and any extra names in the config
-    // (forward-compat for future adapters added before this file is updated).
+    // Compose the known-prefix list from both the hardcoded set and any extra names in the
+    // config (forward-compat for future adapters added before this file is updated).
     const prefixes = [...new Set([...KNOWN_SOURCE_PREFIXES, ...Object.keys(config?.sourceWeights ?? {})])];
-    const stats = (await manifestScan(opts.corpusDir, prefixes)) ?? (await scanSlices(opts.corpusDir, opts.sampleSliceCount ?? 100));
+    const stats = (await manifestScan(opts.corpusDir, prefixes)) ??
+        (await scanParquetFiles(opts.corpusDir, opts.sampleFileCount ?? 100));
     const trainStats = stats.bySplit["train"] ?? {};
     const rows = buildAuditRows(trainStats, config?.sourceWeights ?? {});
     printReport(opts.corpusDir, opts.configPath, stats, rows);

package/out/tools/audit.js.map

@@ -1 +1 @@
-{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../lib/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,eAAe,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC1G,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;GAGG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAgCtC;;;;GAIG;AACH,KAAK,UAAU,WAAW,CAAC,UAAkB;IAC5C,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2B,EAAE,CAAA;IAC1C,IAAI,OAAO,GAAG,KAAK,CAAA;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC,CAAA;IAEpB,KAAK,MAAM,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,MAAM,eAAe,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC;QAC3E,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAA;QAEpE,IAAI,kBAAkB,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAA;YACd,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAE,CAAC,MAAM,CAAA;YAE3C,SAAQ;QACT,CAAC;QAED,IAAI,CAAC,OAAO;YAAE,SAAQ;QAEtB,iCAAiC;QACjC,IAAI,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAQ;QACtC,6FAA6F;QAC7F,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;QAE9C,IAAI,MAAM,IAAI,WAAW,EAAE,CAAC;YAC3B,OAAO,GAAG,KAAK,CAAA;YAEf,SAAQ;QACT,CAAC;QAED,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;QAElD,IAAI,CAAC,EAAE,CAAC;YACP,OAAO,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,CAAC;IACF,CAAC;IAED,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,CAAA;AAClC,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,UAAU,CAAC,SAA0B,EAAE,WAAmB;IACxE,MAAM,KAAK,GAAe,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAExE,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAEvC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAE,SAAQ;QAE3C,MAAM,KAAK,GAAG,CAAC,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE9F,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,2FAA2F;QAC3F,mFAAmF;QACnF,0CAA0C;QAC1C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,wCAAwC;QACxC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,MAAM,CAAA;IAClC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,8FAA8F;IAC9F,8FAA8F;IAC9F,wEAAwE;IACxE,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;GAMG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;GAQG;AACH,KAAK,UAAU,YAAY,CAAC,SAA0B,EAAE,aAAgC;IACvF,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAErD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAElD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAErC,YAAY,CAAC,CAAA;IAEhB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAA;QACzB,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAC,KAAK,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QACpF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC1D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACnE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+FAA+F;IAC/F,kCAAkC;IAClC,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC5D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC9B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEvG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,MAAM;YACN,QAAQ,EAAE,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,8FAA8F;IAC9F,4FAA4F;IAC5F,2EAA2E;IAC3E,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAExC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CACnB,SAA0B,EAC1B,UAA8B,EAC9B,KAAiB,EACjB,IAAgB;IAEhB,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAC3C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QAC1E,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC;gBAClB,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACvB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAA;QAE3E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,CAAA;QAExF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,6DAA6D;gBAC5D,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAe;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1E,8FAA8F;IAC9F,0EAA0E;IAC1E,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEtG,MAAM,KAAK,GACV,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,IAAI,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,gBAAgB,IAAI,GAAG,CAAC,CAAC,CAAA;IAEnH,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}
+{"version":3,"file":"audit.js","sourceRoot":"","sources":["../../lib/tools/audit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,UAAU,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,MAAM,4BAA4B,CAAA;AAC7F,OAAO,EAAE,QAAQ,EAAwB,kBAAkB,EAAE,MAAM,SAAS,CAAA;AAC5E,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,cAAc,EAAE,MAAM,qCAAqC,CAAA;AAEpE;;GAEG;AACH,MAAM,qBAAqB,GAAG,GAAG,CAAA;AAEjC;;;;;GAKG;AACH,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAuCtC;;;;;;;GAOG;AACH,KAAK,UAAU,WAAW,CAAC,UAA2B;IACrD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAEhD,OAAO,EAAE,aAAa,EAAE,cAAc,CAAC,MAAM,iBAAiB,CAAC,UAAU,CAAC,CAAC,CAAC,aAAa,EAAE,CAAA;AAC5F,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAAC,SAA0B,EAAE,WAAmB;IAC9E,MAAM,KAAK,GAAmB,EAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAA;IAE7E,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC;QAC9C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QAErD,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,QAAQ,CAAC,CAAC;YAAE,SAAQ;QAE3C,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;QAEhH,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAA;QAChC,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,WAAW,CAAC,CAAC,CAAA;QACvE,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,WAAW,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,WAAW,CAAC,CAAA;QACnF,MAAM,QAAQ,GAA2B,EAAE,CAAA;QAE3C,yFAAyF;QACzF,kEAAkE;QAClE,wDAAwD;QACxD,8FAA8F;QAC9F,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACzB,MAAM,QAAQ,GAAG,uBAAuB,CAAC,CAAC,CAAC,CAAA;YAC3C,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;QACnD,CAAC;QAED,uCAAuC;QACvC,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAA;QAExD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC;YACvC,QAAQ,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAE,GAAG,KAAK,CAAC,CAAA;QAC/C,CAAC;QAED,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,QAAQ,CAAA;QAC/B,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,MAAM,CAAA;IACnC,CAAC;IAED,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAgB;IAChD,0EAA0E;IAC1E,sFAAsF;IACtF,iDAAiD;IACjD,oDAAoD;IACpD,MAAM,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAA;IAEjE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,SAAS;QAAE,OAAO,CAAC,CAAC,CAAC,CAAC,CAAA;IAExC,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,qBAAqB,GAA0B;IACpD,WAAW;IACX,gBAAgB;IAChB,KAAK;IACL,OAAO;IACP,WAAW;IACX,aAAa;IACb,iBAAiB;IACjB,gBAAgB;IAChB,sBAAsB;IACtB,mBAAmB;IACnB,mBAAmB;IACnB,eAAe;IACf,4DAA4D;IAC5D,qBAAqB;IACrB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;IACxB,wBAAwB;CACxB,CAAA;AAED;;GAEG;AACH,SAAS,YAAY,CAAC,QAAgB,EAAE,aAAgC;IACvE,oEAAoE;IACpE,MAAM,MAAM,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAA;IAEzE,KAAK,MAAM,MAAM,IAAI,MAAM,EAAE,CAAC;QAC7B,IAAI,QAAQ,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,QAAQ,KAAK,MAAM;YAAE,OAAO,MAAM,CAAA;IAC5E,CAAC;IAED,OAAO,WAAW,CAAA;AACnB,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,YAAY,CAC1B,SAA0B,EAC1B,aAAgC;IAEhC,MAAM,YAAY,GAAG,kBAAkB,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IAEnE,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,YAAY,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAElD,MAAM,QAAQ,GAAG,MAAM,iBAAiB,CAErC,YAAY,CAAC,CAAA;IAEhB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAChD,MAAM,OAAO,GAA2C,EAAE,CAAA;IAE1D,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,MAAM,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAA;QACxB,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,IAAI,YAAY,CAAC,IAAI,CAAC,eAAe,IAAI,EAAE,EAAE,aAAa,CAAC,CAAA;QAClF,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAA;QACrB,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAA;IACrD,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE7G,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,CAAA;AAC3D,CAAC;AAWD,SAAS,cAAc,CAAC,KAA6B,EAAE,OAA+B;IACrF,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IACpE,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5E,MAAM,IAAI,GAAe,EAAE,CAAA;IAC3B,+DAA+D;IAC/D,iEAAiE;IACjE,MAAM,aAAa,GAA4B,EAAE,CAAA;IAEjD,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAA;QAC3B,MAAM,SAAS,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAA;QAC3D,aAAa,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC,CAAA;IACrC,CAAC;IAED,MAAM,iBAAiB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;IAEtE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAC7B,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAA;QAClC,MAAM,SAAS,GAAG,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;QAEtG,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,GAAG;YACX,KAAK;YACL,OAAO,EAAE,YAAY,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;YACpD,MAAM;YACN,kBAAkB,EAAE,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG;SACnE,CAAC,CAAA;IACH,CAAC;IAED,wFAAwF;IACxF,qFAAqF;IACrF,yDAAyD;IACzD,mFAAmF;IACnF,sCAAsC;IACtC,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,kBAAkB,KAAK,QAAQ,CAE1E,CAAA;IAED,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB,CAAC,CAAA;IAEnE,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACpB,MAAM,GAAG,GAAG,OAAO,CAAC,CAAC,CAAE,CAAA;QACvB,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,kBAAkB,IAAI,CAAC,CAAA;QAEhD,IACC,GAAG,CAAC,kBAAkB,GAAG,qBAAqB;YAC9C,CAAC,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,kBAAkB,GAAG,IAAI,GAAG,0BAA0B,CAAC,EACvE,CAAC;YACF,GAAG,CAAC,gBAAgB,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,kBAAkB,GAAG,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAA;IAEtC,OAAO,IAAI,CAAA;AACZ,CAAC;AAED,SAAS,SAAS,CAAC,CAAe;IACjC,IAAI,CAAC,KAAK,GAAG;QAAE,OAAO,GAAG,CAAA;IAEzB,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAA;AAClC,CAAC;AAED,SAAS,WAAW,CACnB,SAA0B,EAC1B,UAAuC,EACvC,KAAqB,EACrB,IAAgB;IAEhB,OAAO,CAAC,GAAG,CAAC,oBAAoB,SAAS,EAAE,CAAC,CAAA;IAE5C,IAAI,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,EAAE,CAAC,CAAA;IAC5C,CAAC;IAED,OAAO,CAAC,GAAG,CACV,kBAAkB,KAAK,CAAC,YAAY,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,YAAY,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,UAAU,iBAAiB,CAAC,CAAC,CAAC,EAAE,EAAE,CAC9H,CAAA;IACD,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAEzC,IAAI,UAAU,EAAE,CAAC;QAChB,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAElE,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,QAAQ,CAAC,CAAA;QAC1C,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,OAAO,GAAG,CAAC,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,CAAC,CAAA;QACxE,MAAM,MAAM,GAAG,CAAC,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,CAAA;QACjC,MAAM,MAAM,GAAG,CAAC,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAExF,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAA;QAC5B,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAErD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CACV,MAAM,CAAC;gBACN,GAAG,CAAC,MAAM;gBACV,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC;gBACjB,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC;gBACtB,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;gBAC5D,SAAS,CAAC,GAAG,CAAC,kBAAkB,CAAC;aACjC,CAAC,CACF,CAAA;QACF,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QAEf,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,gBAAgB,KAAK,SAAS,CAAC,CAAA;QAEpE,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,SAAS,CAAC,gBAAgB,CAAA;YACzC,MAAM,SAAS,GAAG,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;YAEhE,OAAO,CAAC,KAAK,CACZ,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,SAAS,CAAC,SAAS,CAAC,kBAAkB,CAAC,GAAG;gBAC9F,qBAAqB,SAAS,uBAAuB;gBACrD,2EAA2E;gBAC3E,4DAA4D,CAC7D,CAAA;QACF,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,GAAG,CAAC,uFAAuF,CAAC,CAAA;QACrG,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,GAAG,IAAI,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,CAAA;QAE1E,IAAI,cAAc,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;YACzC,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,4BAA4B,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC7E,CAAA;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,QAAQ,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAA;QAEvF,IAAI,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CACZ,oEAAoE;gBACnE,oBAAoB,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpE,CAAA;QACF,CAAC;IACF,CAAC;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,KAAK,CAAC,IAAe;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1E,uFAAuF;IACvF,iFAAiF;IACjF,MAAM,QAAQ,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,qBAAqB,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;IAEtG,MAAM,KAAK,GACV,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;QAC9C,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,eAAe,IAAI,GAAG,CAAC,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,CAAA;IAC/C,MAAM,IAAI,GAAG,cAAc,CAAC,UAAU,EAAE,MAAM,EAAE,aAAa,IAAI,EAAE,CAAC,CAAA;IACpE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,IAAI,CAAC,CAAA;AAC1D,CAAC"}

package/out/tools/corpus-stats.d.ts

@@ -5,15 +5,16 @@
  *
  *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
  *
- *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
- *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
- *   as the baseline against which a new slice is compared.
+ *   Reads one or more parquet files, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms. The tool serializes them as JSON. `lint/recipe-output/index.ts` consumes the output file
+ *   as the baseline against which a new recipe output is compared.
  *
  *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
- *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
- *   mainline slice added, a source-pool re-weighted, etc.).
+ *   linter invocations. Re-run this whenever the corpus changes substantially (a new mainline
+ *   recipe output added, a source-pool re-weighted, etc.).
  *
- *   Output schema:
+ *   Output schema (`slice_paths` is the stats file's own key. a stats file already on disk includes it,
+ *   so the linter reads it under that spelling):
  *
  *   ```ts
  *   interface CorpusStats {
@@ -26,19 +27,27 @@
  *   }
  * ```
  *
- *   Usage: node scripts/build-corpus-stats.ts\
- *   --slices <glob-pattern-or-dir>\
- *   --output <stats.json>
+ *   Usage: mailwoman corpus stats\
+ *   --parquet <parquet-file-or-dir>\
+ *   --out <stats.json>
  *
- *   For a quick local-corpus baseline (limited but useful for linter testing): node
- *   scripts/build-corpus-stats.ts\
- *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
- *   --output /tmp/corpus-stats-local.json
+ *   For a quick local-corpus baseline (limited but useful for linter testing): mailwoman corpus stats\
+ *   --parquet $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --out /tmp/corpus-stats-local.json
+ */
+/**
+ * Options for {@linkcode buildCorpusStats}.
  */
 export interface CorpusStatsOptions {
-    slicesArg: string;
+    /**
+     * A directory of parquet files, one parquet file, or a literal path.
+     */
+    parquetPath: string;
     outputPath: string;
-    limitPerSlice?: number;
+    /**
+     * Read at most this many rows per parquet file.
+     */
+    limitPerFile?: number;
 }
 export declare function buildCorpusStats(args: CorpusStatsOptions): Promise<void>;
 //# sourceMappingURL=corpus-stats.d.ts.map

package/out/tools/corpus-stats.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAWH,MAAM,WAAW,kBAAkB;IAClC,SAAS,EAAE,MAAM,CAAA;IACjB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,CAAC,EAAE,MAAM,CAAA;CACtB;AAeD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqE9E"}
+{"version":3,"file":"corpus-stats.d.ts","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAaH;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,WAAW,EAAE,MAAM,CAAA;IACnB,UAAU,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAkBD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC,CAqE9E"}

package/out/tools/corpus-stats.js

@@ -5,15 +5,16 @@
  *
  *   Pre-compute corpus-wide token + bigram label distributions for the corpus linter.
  *
- *   Reads one or more Parquet slices, builds per-(token, label) and per-(bigram, label-bigram)
- *   histograms, and serializes them as JSON. The output file is consumed by `lint-corpus-slice.ts`
- *   as the baseline against which a new slice is compared.
+ *   Reads one or more parquet files, builds per-(token, label) and per-(bigram, label-bigram)
+ *   histograms. The tool serializes them as JSON. `lint/recipe-output/index.ts` consumes the output file
+ *   as the baseline against which a new recipe output is compared.
  *
  *   Stats are cheap to compute (~5–30s per 100K rows) but expensive enough that we cache them between
- *   linter invocations. Re-run this script whenever the corpus changes substantially (a new
- *   mainline slice added, a source-pool re-weighted, etc.).
+ *   linter invocations. Re-run this whenever the corpus changes substantially (a new mainline
+ *   recipe output added, a source-pool re-weighted, etc.).
  *
- *   Output schema:
+ *   Output schema (`slice_paths` is the stats file's own key. a stats file already on disk includes it,
+ *   so the linter reads it under that spelling):
  *
  *   ```ts
  *   interface CorpusStats {
@@ -26,41 +27,44 @@
  *   }
  * ```
  *
- *   Usage: node scripts/build-corpus-stats.ts\
- *   --slices <glob-pattern-or-dir>\
- *   --output <stats.json>
+ *   Usage: mailwoman corpus stats\
+ *   --parquet <parquet-file-or-dir>\
+ *   --out <stats.json>
  *
- *   For a quick local-corpus baseline (limited but useful for linter testing): node
- *   scripts/build-corpus-stats.ts\
- *   --slices $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
- *   --output /tmp/corpus-stats-local.json
+ *   For a quick local-corpus baseline (limited but useful for linter testing): mailwoman corpus stats\
+ *   --parquet $MAILWOMAN_DATA_ROOT/corpus/versioned/v0.4.0/corpus-v0.4.0/train/\
+ *   --out /tmp/corpus-stats-local.json
  */
 import { ByteFormatter } from "@mailwoman/core/fs/formatters";
-import { readDirectory, statPath } from "@mailwoman/core/fs/readers";
+import { statPath } from "@mailwoman/core/fs/readers";
 import { writeLocalJSONFile } from "@mailwoman/core/fs/writers";
-import { join } from "path-ts";
-import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/slice-stats";
+import { stringifyJSON } from "@mailwoman/core/json";
+import { PathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { accumulateCooccurrences, createCooccurrenceStats, streamTokenLabelRows } from "#utils/cooccurrence-stats";
 const MIN_BIGRAM_COUNT = 2;
-async function discoverSlices(slicesArg) {
-    const stat = await statPath(slicesArg);
+async function discoverParquetFiles(pathArg) {
+    const stat = await statPath(pathArg);
     if (stat.isDirectory()) {
-        return (await readDirectory(slicesArg)).filter((f) => f.endsWith(".parquet")).map((f) => join(slicesArg, f));
+        const root = PathBuilder.from(pathArg);
+        const names = await Globerator.files("parquet", { cwd: root, recursive: false }).toSorted();
+        return names.map((name) => root(name).toString());
     }
-    if (stat.isFile() && slicesArg.endsWith(".parquet"))
-        return [slicesArg];
+    if (stat.isFile() && pathArg.endsWith(".parquet"))
+        return [pathArg];
     // Otherwise treat as a literal path list (one per line if it's stdin-friendly).
-    return [slicesArg];
+    return [pathArg];
 }
 export async function buildCorpusStats(args) {
-    const slicePaths = await discoverSlices(args.slicesArg);
-    console.error(`Discovered ${slicePaths.length} parquet slice(s)`);
+    const parquetPaths = await discoverParquetFiles(args.parquetPath);
+    console.error(`Discovered ${parquetPaths.length} parquet file(s)`);
     const stats = createCooccurrenceStats();
     const { tokens: tokenStats, bigrams: bigramStats } = stats;
     let totalRows = 0;
-    for (const path of slicePaths) {
+    for (const path of parquetPaths) {
         console.error(`Reading ${path}...`);
         const before = totalRows;
-        for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerSlice)) {
+        for await (const { tokens, labels } of streamTokenLabelRows(path, args.limitPerFile)) {
             totalRows++;
             if (tokens.length !== labels.length)
                 continue; // skip malformed
@@ -68,9 +72,9 @@ export async function buildCorpusStats(args) {
         }
         console.error(`  ${totalRows - before} rows; running totals: ${tokenStats.size} unique tokens, ${bigramStats.size} unique bigrams`);
     }
-    // Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane. Token stats
-    // stay complete — they're cheap and we need accuracy at the long tail for label-vacuum
-    // detection.
+    // Prune bigrams below MIN_BIGRAM_COUNT to keep the output file size sane.
+    // Token stats stay complete.
+    // They're cheap and we need accuracy at the long tail for label-vacuum detection.
     let prunedBigrams = 0;
     for (const [k, labelMap] of bigramStats) {
         let total = 0;
@@ -85,7 +89,7 @@ export async function buildCorpusStats(args) {
     console.error(`Pruned ${prunedBigrams} singleton bigrams; ${bigramStats.size} remain`);
     const out = {
         row_count: totalRows,
-        slice_paths: slicePaths,
+        slice_paths: parquetPaths,
         tokens: {},
         bigrams: {},
     };
@@ -96,7 +100,7 @@ export async function buildCorpusStats(args) {
         out.bigrams[k] = Object.fromEntries(labelMap);
     }
     await writeLocalJSONFile(out, args.outputPath);
-    const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(JSON.stringify(out)));
+    const formattedSize = ByteFormatter.formatIEC(Buffer.byteLength(stringifyJSON(out)));
     console.error(`Wrote ${args.outputPath} (${formattedSize}) — ${totalRows} rows, ${tokenStats.size} tokens, ${bigramStats.size} bigrams`);
 }
 //# sourceMappingURL=corpus-stats.js.map

package/out/tools/corpus-stats.js.map

@@ -1 +1 @@
-{"version":3,"file":"corpus-stats.js","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAE9B,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAE3G,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAQ1B,KAAK,UAAU,cAAc,CAAC,SAAiB;IAC9C,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;IAEtC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,OAAO,CAAC,MAAM,aAAa,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAA;IAC7G,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,SAAS,CAAC,CAAA;IAEvE,gFAAgF;IAChF,OAAO,CAAC,SAAS,CAAC,CAAA;AACnB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAwB;IAC9D,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IAEvD,OAAO,CAAC,KAAK,CAAC,cAAc,UAAU,CAAC,MAAM,mBAAmB,CAAC,CAAA;IAEjE,MAAM,KAAK,GAAG,uBAAuB,EAAE,CAAA;IACvC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,KAAK,CAAA;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC/B,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,CAAA;QAEnC,MAAM,MAAM,GAAG,SAAS,CAAA;QAExB,IAAI,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YACvF,SAAS,EAAE,CAAA;YAEX,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,SAAQ,CAAC,iBAAiB;YAE/D,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/C,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,KAAK,SAAS,GAAG,MAAM,0BAA0B,UAAU,CAAC,IAAI,mBAAmB,WAAW,CAAC,IAAI,iBAAiB,CACpH,CAAA;IACF,CAAC;IAED,sFAAsF;IACtF,uFAAuF;IACvF,aAAa;IACb,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAA;QACX,CAAC;QAED,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YAC9B,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAErB,aAAa,EAAE,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,aAAa,uBAAuB,WAAW,CAAC,IAAI,SAAS,CAAC,CAAA;IAEtF,MAAM,GAAG,GAAG;QACX,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,UAAU;QACvB,MAAM,EAAE,EAA4C;QACpD,OAAO,EAAE,EAA4C;KACrD,CAAA;IAED,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IAC9C,MAAM,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAErF,OAAO,CAAC,KAAK,CACZ,SAAS,IAAI,CAAC,UAAU,KAAK,aAAa,OAAO,SAAS,UAAU,UAAU,CAAC,IAAI,YAAY,WAAW,CAAC,IAAI,UAAU,CACzH,CAAA;AACF,CAAC"}
+{"version":3,"file":"corpus-stats.js","sourceRoot":"","sources":["../../lib/tools/corpus-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAoCG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,+BAA+B,CAAA;AAC7D,OAAO,EAAE,QAAQ,EAAE,MAAM,4BAA4B,CAAA;AACrD,OAAO,EAAE,kBAAkB,EAAE,MAAM,4BAA4B,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAE,uBAAuB,EAAE,uBAAuB,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAA;AAElH,MAAM,gBAAgB,GAAG,CAAC,CAAA;AAiB1B,KAAK,UAAU,oBAAoB,CAAC,OAAe;IAClD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAA;IAEpC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACxB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtC,MAAM,KAAK,GAAG,MAAM,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,GAAG,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAA;QAE3F,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAA;IAClD,CAAC;IAED,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,OAAO,CAAC,CAAA;IAEnE,gFAAgF;IAChF,OAAO,CAAC,OAAO,CAAC,CAAA;AACjB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAwB;IAC9D,MAAM,YAAY,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAEjE,OAAO,CAAC,KAAK,CAAC,cAAc,YAAY,CAAC,MAAM,kBAAkB,CAAC,CAAA;IAElE,MAAM,KAAK,GAAG,uBAAuB,EAAE,CAAA;IACvC,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,KAAK,CAAA;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;QACjC,OAAO,CAAC,KAAK,CAAC,WAAW,IAAI,KAAK,CAAC,CAAA;QAEnC,MAAM,MAAM,GAAG,SAAS,CAAA;QAExB,IAAI,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,oBAAoB,CAAC,IAAI,EAAE,IAAI,CAAC,YAAY,CAAC,EAAE,CAAC;YACtF,SAAS,EAAE,CAAA;YAEX,IAAI,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM;gBAAE,SAAQ,CAAC,iBAAiB;YAE/D,uBAAuB,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;QAC/C,CAAC;QAED,OAAO,CAAC,KAAK,CACZ,KAAK,SAAS,GAAG,MAAM,0BAA0B,UAAU,CAAC,IAAI,mBAAmB,WAAW,CAAC,IAAI,iBAAiB,CACpH,CAAA;IACF,CAAC;IAED,0EAA0E;IAC1E,6BAA6B;IAC7B,kFAAkF;IAClF,IAAI,aAAa,GAAG,CAAC,CAAA;IAErB,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,GAAG,CAAC,CAAA;QAEb,KAAK,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;YACnC,KAAK,IAAI,CAAC,CAAA;QACX,CAAC;QAED,IAAI,KAAK,GAAG,gBAAgB,EAAE,CAAC;YAC9B,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;YAErB,aAAa,EAAE,CAAA;QAChB,CAAC;IACF,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,UAAU,aAAa,uBAAuB,WAAW,CAAC,IAAI,SAAS,CAAC,CAAA;IAEtF,MAAM,GAAG,GAAG;QACX,SAAS,EAAE,SAAS;QACpB,WAAW,EAAE,YAAY;QACzB,MAAM,EAAE,EAA4C;QACpD,OAAO,EAAE,EAA4C;KACrD,CAAA;IAED,KAAK,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,UAAU,EAAE,CAAC;QACzC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,KAAK,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,UAAU,CAAC,CAAA;IAC9C,MAAM,aAAa,GAAG,aAAa,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;IAEpF,OAAO,CAAC,KAAK,CACZ,SAAS,IAAI,CAAC,UAAU,KAAK,aAAa,OAAO,SAAS,UAAU,UAAU,CAAC,IAAI,YAAY,WAAW,CAAC,IAAI,UAAU,CACzH,CAAA;AACF,CAAC"}