@mailwoman/corpus 9.1.0 → 9.3.0

package/out/src/tools/jsonl-to-parquet.js

@@ -1,164 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Convert a JSONL of LabeledRow objects to a Parquet shard matching the v0.5.0 schema.
- *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the shard. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads shards by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base shards this overlay
- *   rides alongside.
- *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
- *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the shard. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
- */
-import { randomUUID } from "node:crypto";
-import { createWriteStream } from "node:fs";
-import { unlink } from "node:fs/promises";
-import { tmpdir } from "node:os";
-import { join } from "node:path";
-import { parseJSONStrict } from "@mailwoman/core/objects";
-import { TextSpliterator } from "spliterator";
-const REQUIRED_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
-/**
- * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
- *
- * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
- * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
- * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
- * round-trips as a plain integer where INT64 would surface as bigint.
- */
-const COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
-/**
- * Enforce the #519 span contract per row: all three present, parallel lengths.
- *
- * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
- */
-function assertSpanTriple(row, lineNo) {
-    const present = SPAN_COLUMNS.filter((c) => row[c] != null);
-    if (present.length !== SPAN_COLUMNS.length) {
-        const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
-        throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
-            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
-            "must carry span_starts/span_ends/span_tags; re-emit this shard through alignRow.");
-    }
-    const n = row.span_starts.length;
-    if (row.span_ends.length !== n || row.span_tags.length !== n) {
-        throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
-            `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
-            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
-    }
-}
-/**
- * Escape a path for single-quoted SQL string literals.
- */
-function sqlString(value) {
-    return value.replaceAll("'", "''");
-}
-/**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet shard.
- */
-export async function jsonlToParquet(options, report) {
-    const rowGroupSize = options.rowGroupSize ?? 50_000;
-    if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
-        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
-    }
-    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
-    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
-    // `finally` covers the validation pass too, so a mid-stream span-triple failure leaves no orphan.
-    const stagePath = join(tmpdir(), `mw-jsonl-to-parquet-${randomUUID()}.ndjson`);
-    const stage = createWriteStream(stagePath, { encoding: "utf8" });
-    try {
-        let rows = 0;
-        let lineNo = 0;
-        // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
-        // DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
-        // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
-        // same as readline's crlfDelay:Infinity did.
-        for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
-            lineNo++;
-            const line = rawLine.trim();
-            if (!line)
-                continue;
-            const row = parseJSONStrict(line);
-            assertSpanTriple(row, lineNo);
-            // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
-            // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
-            stage.write(line + "\n");
-            rows++;
-        }
-        await new Promise((resolve, reject) => {
-            stage.end((err) => (err ? reject(err) : resolve()));
-        });
-        report?.(`Read ${rows} rows from ${options.input}`);
-        const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
-        const selectList = REQUIRED_COLUMNS.join(", ");
-        // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
-        const { DuckDBInstance } = await import("@duckdb/node-api");
-        const instance = await DuckDBInstance.create();
-        const db = await instance.connect();
-        // Row order is load-bearing: the overlay-manifest assembler records first/last source_id from
-        // shard order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
-        await db.run("SET preserve_insertion_order=true");
-        await db.run(`COPY (SELECT ${selectList} FROM read_json('${sqlString(stagePath)}', ` +
-            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-            `TO '${sqlString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
-        const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${sqlString(options.output)}')`);
-        const written = Number(counted.getRowObjects()[0].n);
-        report?.(`Wrote ${written} rows to ${options.output}`);
-        return { read: rows, written, outPath: options.output };
-    }
-    finally {
-        stage.destroy();
-        await unlink(stagePath).catch(() => { });
-    }
-}
-//# sourceMappingURL=jsonl-to-parquet.js.map

package/out/src/tools/jsonl-to-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../../src/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAA;AAC3C,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAA;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAA;AACzD,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,kGAAkG;IAClG,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,uBAAuB,UAAU,EAAE,SAAS,CAAC,CAAA;IAC9E,MAAM,KAAK,GAAG,iBAAiB,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAA;IAEhE,IAAI,CAAC;QACJ,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,IAAI,MAAM,GAAG,CAAC,CAAA;QAEd,8FAA8F;QAC9F,gGAAgG;QAChG,6FAA6F;QAC7F,6CAA6C;QAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YACtE,MAAM,EAAE,CAAA;YACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;YAE3B,IAAI,CAAC,IAAI;gBAAE,SAAQ;YACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;YAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YAC7B,6FAA6F;YAC7F,yFAAyF;YACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;YAExB,IAAI,EAAE,CAAA;QACP,CAAC;QAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;QACnE,CAAC,CAAC,CAAA;QAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;QAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;QACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE9C,gFAAgF;QAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;QAC9C,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QACnC,8FAA8F;QAC9F,6FAA6F;QAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,SAAS,CAAC,SAAS,CAAC,KAAK;YACtE,aAAa,cAAc,mCAAmC;YAC9D,OAAO,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACzG,CAAA;QAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;QAChH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;QACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;QAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;IACxD,CAAC;YAAS,CAAC;QACV,KAAK,CAAC,OAAO,EAAE,CAAA;QACf,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;IACxC,CAAC;AACF,CAAC"}

package/out/src/tools/lint-shard-vocab.d.ts

@@ -1,90 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
- *
- *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
- *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
- */
-/**
- * Options for {@linkcode lintShardVocab}.
- */
-export interface LintShardVocabOptions {
-    /**
-     * The shard parquet to lint.
-     */
-    shard: string;
-    /**
-     * Base corpus version. Default `v0.5.0`.
-     */
-    baseVersion?: string;
-    /**
-     * Base corpus root. Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
-     */
-    baseRoot?: string;
-    /**
-     * Base-majority confidence floor for a contradiction. Default 0.7.
-     */
-    threshold?: number;
-    /**
-     * Minimum base support to judge a token. Default 50.
-     */
-    minCount?: number;
-    /**
-     * Fraction of base parts to scan (proportional per-source slice below 1.0). Default 1.0.
-     */
-    fraction?: number;
-}
-/**
- * One contradiction row: token, shard tag, base tag, base fraction, base total.
- */
-export type ShardVocabRow = [token: string, shardTag: string, baseTag: string, baseFrac: number, baseTotal: number];
-/**
- * Findings summary returned by {@linkcode lintShardVocab}.
- */
-export interface LintShardVocabSummary {
-    /**
-     * Real contradictions — the command exits 1 when nonzero.
-     */
-    errors: number;
-    /**
-     * Affix-split rows (EXPECTED — the loader's affix-relabel handles them).
-     */
-    warnings: number;
-    findings: {
-        contradictions: ShardVocabRow[];
-        affixSplits: ShardVocabRow[];
-    };
-}
-/**
- * Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
- */
-export declare function lintShardVocab(options: LintShardVocabOptions): Promise<LintShardVocabSummary>;
-//# sourceMappingURL=lint-shard-vocab.d.ts.map

package/out/src/tools/lint-shard-vocab.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard-vocab.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AA8KH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAEnH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,aAAa,EAAE,CAAC;QAAC,WAAW,EAAE,aAAa,EAAE,CAAA;KAAE,CAAA;CAC3E;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,OAAO,EAAE,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAmJnG"}

package/out/src/tools/lint-shard-vocab.js

@@ -1,281 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   #511 base-consistency lint, GENERALIZED + COUNTRY-SCOPED (v2) — any synthetic shard vs the base.
- *
- *   Ported from scripts/lint-shard-vocab.py (pyarrow → @duckdb/node-api); behavior preserved
- *   byte-for-byte (same flags, same stdout, same verdicts). The base-root default routes through
- *   `dataRootPath` so the lab `/mnt/playpen` literal stays in its one home
- *   (core/utils/data-root.ts) and `$MAILWOMAN_DATA_ROOT` is honored; with the env unset it equals
- *   the Python default.
- *
- *   The #511 lesson: a synthetic shard must not label a token a tag the BASE dominantly labels
- *   something else, or training gets conflicting gradients on the same token and the minority (the
- *   shard) loses. This reads a shard's own (token -> tag) and checks each token against the base.
- *
- *   WHY v2 IS COUNTRY-SCOPED + FULL-COUNT (the night-2026-06-18 lesson, learned the hard way over
- *   three tries): a token's correct tag is COUNTRY-specific — "Paris" is locality in FR data and
- *   street in US "Paris Ave"; "Marion" is a US town AND many US "Marion" streets. So:
- *
- *   1. A cross-COUNTRY aggregate mis-judges any country-specific token (v1 uniform AND a proportional
- *        retry both false-flagged FR cities as "street" from US street-contexts).
- *   2. A SMALL sample is street-BIASED regardless, because the street sources (tiger 39 + nad 378 parts)
- *        dwarf the locality sources (a small US-scoped spot-check read Indianapolis 54% street vs
- *        its true 219700:29 LOCALITY). The fix: tally each shard token's base tag SCOPED to the
- *        country the shard uses it in (the base has a `country` column), over a LARGE/FULL scan
- *        (`fraction`, default 1.0). Pure-numeric tokens excluded (house_number/postcode are
- *        context-determined). An affix-split flag (shard street_suffix/_prefix vs base "street") is
- *        EXPECTED — the loader's affix-relabel handles it; weigh those separately.
- *
- *   Usage: mailwoman dev lint shard-vocab --shard <shard.parquet>
- *   [--base-version v0.5.0] [--base-root <dir>] [--fraction 1.0] [--threshold 0.7] [--min-count
- *   50]
- */
-import { readdirSync } from "node:fs";
-import { join } from "node:path";
-import { tryParsingJSON } from "@mailwoman/core/objects";
-import { dataRootPath } from "@mailwoman/core/utils";
-/**
- * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
- */
-function stripBIO(label) {
-    const head = label.slice(0, 2);
-    return head === "B-" || head === "I-" ? label.slice(2) : label;
-}
-/**
- * Python `str.isdigit()`: non-empty and every character a Unicode digit. Pure-numeric tokens (house_number / postcode)
- * are context-determined, not lexical vocab, so they're excluded. `\p{Nd}` covers the decimal digits these address
- * corpora actually contain.
- */
-function isDigit(token) {
-    return token.length > 0 && /^\p{Nd}+$/u.test(token);
-}
-/**
- * Round half to even (banker's rounding) — Python's built-in `round()` and `format(..., ".0%")` both use it, so percent
- * strings and the proportional `fraction` slice match the Python output exactly.
- */
-function pyRound(x) {
-    const floor = Math.floor(x);
-    const diff = x - floor;
-    if (diff < 0.5)
-        return floor;
-    if (diff > 0.5)
-        return floor + 1;
-    return floor % 2 === 0 ? floor : floor + 1;
-}
-/**
- * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
- */
-function pct(frac) {
-    return `${pyRound(frac * 100)}%`;
-}
-/**
- * Format a float the way a Python f-string renders it: integer-valued floats keep one decimal (1.0 -> "1.0"),
- * everything else is its shortest decimal (0.5 -> "0.5"). Used for the `fraction` echo so the banner matches the Python
- * print.
- */
-function pyFloat(n) {
-    return Number.isInteger(n) ? n.toFixed(1) : String(n);
-}
-/**
- * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
- */
-function pad(value, width) {
-    return value.padEnd(width);
-}
-/**
- * The dominant tag of a counter: (tag, total, fraction). Empty counter -> ("", 0, 0.0). Ties go to the first-inserted
- * tag, mirroring `Counter.most_common(1)` (stable on equal counts).
- */
-function dominant(counter) {
-    let total = 0;
-    let bestTag = "";
-    let bestCount = -1;
-    for (const [tag, count] of counter) {
-        total += count;
-        if (count > bestCount) {
-            bestCount = count;
-            bestTag = tag;
-        }
-    }
-    if (total === 0)
-        return ["", 0, 0];
-    return [bestTag, total, bestCount / total];
-}
-/**
- * Bump a (key -> count) tally, creating the inner counter on first sight.
- */
-function bump(table, key, sub) {
-    let counter = table.get(key);
-    if (!counter) {
-        counter = new Map();
-        table.set(key, counter);
-    }
-    counter.set(sub, (counter.get(sub) ?? 0) + 1);
-}
-/**
- * Read a corpus parquet into rows, projecting only tokens/labels/country. The list columns ride out as JSON text
- * (DuckDB `to_json`) — the same trick the gazetteer builders use for nested columns — and parse back to string arrays
- * here.
- */
-async function readRows(con, path) {
-    const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
-    const raw = result.getRowObjects();
-    const rows = [];
-    for (const r of raw) {
-        const tokens = tryParsingJSON(String(r.tokens));
-        const labels = tryParsingJSON(String(r.labels));
-        if (!Array.isArray(tokens) || !Array.isArray(labels))
-            continue;
-        rows.push({
-            tokens: tokens,
-            labels: labels,
-            country: r.country == null ? null : String(r.country),
-        });
-    }
-    return rows;
-}
-/**
- * Read just the first row's `source` value — used to group base parts for a proportional slice.
- */
-async function readSource(con, path) {
-    const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
-    const rows = result.getRowObjects();
-    return rows.length ? String(rows[0].source) : "";
-}
-/**
- * Non-recursive `*.parquet` glob, sorted lexicographically — the Python `sorted(glob.glob(...))`.
- */
-function globParquet(dir) {
-    let names;
-    try {
-        names = readdirSync(dir);
-    }
-    catch {
-        return [];
-    }
-    return names
-        .filter((f) => f.endsWith(".parquet"))
-        .map((f) => join(dir, f))
-        .toSorted();
-}
-/**
- * Lint a synthetic shard's (token → tag) vocabulary against the base corpus, country-scoped.
- */
-export async function lintShardVocab(options) {
-    const baseVersion = options.baseVersion ?? "v0.5.0";
-    const baseRoot = options.baseRoot ?? dataRootPath("corpus", "versioned");
-    const threshold = options.threshold ?? 0.7;
-    const minCount = options.minCount ?? 50;
-    const fraction = options.fraction ?? 1;
-    // @duckdb/node-api is an optional peer — lazy import (the pipeline convention).
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const instance = await DuckDBInstance.create();
-    const con = await instance.connect();
-    // 1. the shard's own (token -> dominant tag) + the COUNTRIES it uses each token in
-    const shardRows = await readRows(con, options.shard);
-    const shardTags = new Map();
-    const shardCountries = new Map();
-    for (const { tokens, labels, country } of shardRows) {
-        const n = Math.min(tokens.length, labels.length);
-        for (let i = 0; i < n; i++) {
-            const w = tokens[i];
-            const l = labels[i];
-            if (isDigit(w))
-                continue; // numbers are context-determined (house_number/postcode), not lexical vocab
-            bump(shardTags, w, stripBIO(l));
-            let set = shardCountries.get(w);
-            if (!set) {
-                set = new Set();
-                shardCountries.set(w, set);
-            }
-            set.add(country);
-        }
-    }
-    const shardVocab = new Set(shardTags.keys());
-    console.log(`shard: ${shardRows.length} rows, ${shardVocab.size} unique tokens`);
-    // 2. base parts — FULL by default; fraction<1 takes a proportional per-source slice (still big)
-    const trainDir = join(baseRoot, baseVersion, `corpus-${baseVersion}`, "train");
-    let parts = globParquet(trainDir);
-    if (!parts.length) {
-        throw new Error("no base parts found");
-    }
-    if (fraction < 1) {
-        const bysrc = new Map();
-        for (const p of parts) {
-            const src = await readSource(con, p);
-            let list = bysrc.get(src);
-            if (!list) {
-                list = [];
-                bysrc.set(src, list);
-            }
-            list.push(p);
-        }
-        const sliced = [];
-        for (const ps of bysrc.values()) {
-            const take = Math.max(2, pyRound(ps.length * fraction));
-            for (const p of ps.slice(0, take)) {
-                sliced.push(p);
-            }
-        }
-        parts = sliced;
-    }
-    console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${pyFloat(fraction)}), COUNTRY-scoped`);
-    // 3. tally each shard token's base tag, SCOPED to the country the shard uses it in
-    const baseTags = new Map();
-    for (let i = 0; i < parts.length; i++) {
-        const rows = await readRows(con, parts[i]);
-        for (const { tokens, labels, country } of rows) {
-            const n = Math.min(tokens.length, labels.length);
-            for (let j = 0; j < n; j++) {
-                const w = tokens[j];
-                if (shardVocab.has(w) && shardCountries.get(w).has(country)) {
-                    bump(baseTags, w, stripBIO(labels[j]));
-                }
-            }
-        }
-        if ((i + 1) % 100 === 0) {
-            console.log(`  ...${i + 1}/${parts.length} parts`);
-        }
-    }
-    // 4. compare; flag contradictions (affix-split is expected — surfaced but tagged)
-    const flagged = [];
-    const affix = [];
-    for (const w of shardVocab) {
-        const [sTag] = dominant(shardTags.get(w));
-        const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
-        if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
-            continue;
-        const row = [w, sTag, bTag, bFrac, bTotal];
-        if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
-            affix.push(row);
-        }
-        else {
-            flagged.push(row);
-        }
-    }
-    const sections = [
-        ["CONTRADICTION", flagged],
-        ["affix-split (EXPECTED — affix-relabel handles)", affix],
-    ];
-    for (const [label, rows] of sections) {
-        if (!rows.length)
-            continue;
-        rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
-        console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
-        for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
-            console.log(`  ${pad(w, 18)} shard=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
-        }
-    }
-    if (!flagged.length) {
-        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — shard base-consistent`);
-    }
-    return {
-        errors: flagged.length,
-        warnings: affix.length,
-        findings: { contradictions: flagged, affixSplits: affix },
-    };
-}
-//# sourceMappingURL=lint-shard-vocab.js.map

package/out/src/tools/lint-shard-vocab.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard-vocab.js","sourceRoot":"","sources":["../../../src/tools/lint-shard-vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkCG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAEhC,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAA;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAA;AAWpD;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;;GAGG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,CAAA;IAEtB,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,CAAA;IAE5B,IAAI,IAAI,GAAG,GAAG;QAAE,OAAO,KAAK,GAAG,CAAC,CAAA;IAEhC,OAAO,KAAK,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;GAIG;AACH,SAAS,OAAO,CAAC,CAAS;IACzB,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AASD;;;;GAIG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAW;IAC/B,IAAI,KAAe,CAAA;IAEnB,IAAI,CAAC;QACJ,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,EAAE,CAAA;IACV,CAAC;IAED,OAAO,KAAK;SACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;SACrC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;SACxB,QAAQ,EAAE,CAAA;AACb,CAAC;AAoDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,OAA8B;IAClE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAA;IACxE,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,gFAAgF;IAChF,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAC9C,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IAEpC,mFAAmF;IACnF,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,KAAK,CAAC,CAAA;IACpD,MAAM,SAAS,GAAG,IAAI,GAAG,EAA+B,CAAA;IACxD,MAAM,cAAc,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE5D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,SAAS,EAAE,CAAC;QACrD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ,CAAC,4EAA4E;YACrG,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAC/B,IAAI,GAAG,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAE/B,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,cAAc,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC3B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAA;IAE5C,OAAO,CAAC,GAAG,CAAC,UAAU,SAAS,CAAC,MAAM,UAAU,UAAU,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAEhF,gGAAgG;IAChG,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAC9E,IAAI,KAAK,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAA;QAE3B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACf,CAAC;QACF,CAAC;QAED,KAAK,GAAG,MAAM,CAAA;IACf,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,CAAA;IAElH,mFAAmF;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,cAAc,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC9D,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,kFAAkF;IAClF,MAAM,OAAO,GAAoB,EAAE,CAAA;IACnC,MAAM,KAAK,GAAoB,EAAE,CAAA;IAEjC,KAAK,MAAM,CAAC,IAAI,UAAU,EAAE,CAAC;QAC5B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC1C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAkB,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEzD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAqC;QAClD,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QAChG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,2BAA2B,CACvH,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}

package/out/src/tools/lint-shard.d.ts

@@ -1,93 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Corpus linter. Compares a new shard against pre-computed corpus statistics and flags patterns
- *   that would cause the class of failure we hit with v0.6.2's "5th Avenue Theatre" adversarial
- *   venue templates.
- *
- *   Per DeepSeek turn 9 design (2026-05-29). v1 checks:
- *
- *   1. **Token-label distribution outliers.** For each token in the new shard, compare the shard's
- *        majority label to the corpus's majority label. Flag when the corpus has a
- *        confidently-established majority (>66%) AND the shard's majority differs AND both have
- *        non-trivial counts (shard ≥ 50, corpus ≥ 200).
- *   2. **Label-vacuum tokens.** Token labeled with a tag that has ZERO instances in the corpus for that
- *        token, despite the token being well-represented in the corpus. Stronger signal than #1 —
- *        we're introducing a novel association, not shifting a distribution.
- *   3. **Bigram-label collisions.** Identical (token_bigram, label_bigram) appears in shard while the
- *        same token_bigram has a DIFFERENT majority label_bigram in the corpus. The "5th Avenue"
- *        with [B-venue, I-venue] vs corpus's [B-house_number, I-street] case.
- *   4. **Common-form anti-pattern rules.** Applies `lint-rules.json` — token-regex → forbidden-labels
- *        mappings — flagging matches.
- *   5. **Basic sanity.** Truncated rows (tokens.length !== labels.length), all-O rows >90% of shard.
- *
- *   Output: markdown report on stdout, optional JSON sidecar via `outJSON`. The command exits 0 if
- *   no errors, 1 if any errors (warnings don't gate). Per the design, the MANIFEST entry for a
- *   flagged shard should require `lint_acknowledged: true` before training consumes it.
- *
- *   Usage: mailwoman dev lint corpus-shard\
- *   --shard <new-shard.parquet>\
- *   --stats <corpus-stats.json>\
- *   [--rules <rules.json>]\
- *   [--out-md /tmp/lint-report.md]\
- *   [--out-json /tmp/lint-report.json]
- */
-/**
- * Options for {@linkcode lintCorpusShard}.
- */
-export interface LintCorpusShardOptions {
-    /**
-     * The new shard parquet to lint.
-     */
-    shardPath: string;
-    /**
-     * Pre-computed corpus stats JSON (see `corpus-stats.ts`).
-     */
-    statsPath: string;
-    /**
-     * Anti-pattern rules JSON. Default: the `lint-rules.json` beside this module.
-     */
-    rulesPath?: string;
-    /**
-     * Write the markdown report here as well as stdout.
-     */
-    outMd?: string;
-    /**
-     * Write a JSON sidecar of the flags + summary here.
-     */
-    outJSON?: string;
-}
-/**
- * One lint flag emitted by a check.
- */
-export interface LintShardFlag {
-    check: string;
-    severity: "error" | "warn";
-    token?: string;
-    bigram?: string;
-    shardLabel?: string;
-    corpusLabel?: string;
-    shardCount?: number;
-    corpusCount?: number;
-    detail: string;
-    ruleID?: string;
-}
-/**
- * Findings summary returned by {@linkcode lintCorpusShard}.
- */
-export interface LintCorpusShardSummary {
-    errors: number;
-    warnings: number;
-    findings: LintShardFlag[];
-    /**
-     * The rendered markdown report (also printed to stdout).
-     */
-    report: string;
-}
-/**
- * Lint a shard against corpus stats + the anti-pattern rules; print the markdown report to stdout.
- */
-export declare function lintCorpusShard(options: LintCorpusShardOptions, report?: (line: string) => void): Promise<LintCorpusShardSummary>;
-//# sourceMappingURL=lint-shard.d.ts.map

package/out/src/tools/lint-shard.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"lint-shard.d.ts","sourceRoot":"","sources":["../../../src/tools/lint-shard.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AA+CH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,EAAE,MAAM,CAAA;IACjB;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;CAChB;AA6HD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,OAAO,GAAG,MAAM,CAAA;IAC1B,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE,aAAa,EAAE,CAAA;IACzB;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;CACd;AAqND;;GAEG;AACH,wBAAsB,eAAe,CACpC,OAAO,EAAE,sBAAsB,EAC/B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,sBAAsB,CAAC,CAiEjC"}