@mailwoman/corpus 9.4.0 → 10.1.0

package/out/tools/jsonl-to-parquet.d.ts

@@ -3,46 +3,33 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ * Convert a jsonl of LabeledRow objects to a Parquet file matching the schema the corpus writes.
  *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
- *   rides alongside.
+ * DuckDB reproduces the PyArrow file's logical schema and column order. A PyArrow reader therefore sees an identical table.
+ * The trainer reads Parquet by column name, so physical layout does not affect it.
  *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
+ * Schema: `PARQUET_COLUMNS` from `#parquet/schema`, the same list the native writer uses.
  *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
+ * Every row must contain the span triple. A row without it came from a producer that has not migrated.
+ * The writer would silently drop the character-offset labels if it wrote that row. Fail loudly and report the row number.
  */
+import type { PathBuilderLike } from "path-ts";
 /**
  * Options for {@linkcode jsonlToParquet}.
  */
 export interface JSONLToParquetOptions {
     /**
-     * The labeled-row JSONL to convert.
+     * The labeled-row jsonl to convert.
      */
-    input: string;
+    input: PathBuilderLike;
     /**
-     * The parquet slice to write.
+     * The parquet file to write.
      */
-    output: string;
+    output: PathBuilderLike;
     /**
-     * Parquet row-group size. Default 50000.
+     * Parquet row-group size.
+     *
+     * Default 50000.
      */
     rowGroupSize?: number;
 }
@@ -55,7 +42,7 @@ export interface JSONLToParquetSummary {
     outPath: string;
 }
 /**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ * Convert a labeled-row jsonl to a v0.5.0-schema Parquet file.
  */
 export declare function jsonlToParquet(options: JSONLToParquetOptions, report?: (line: string) => void): Promise<JSONLToParquetSummary>;
 //# sourceMappingURL=jsonl-to-parquet.d.ts.map

package/out/tools/jsonl-to-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAsDH;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,MAAM,CAAA;IACb;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA8BD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA6DhC"}
+{"version":3,"file":"jsonl-to-parquet.d.ts","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAMH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAsB9C;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC;;OAEG;IACH,KAAK,EAAE,eAAe,CAAA;IACtB;;OAEG;IACH,MAAM,EAAE,eAAe,CAAA;IACvB;;;;OAIG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;GAEG;AACH,MAAM,WAAW,qBAAqB;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,MAAM,CAAA;CACf;AA8BD;;GAEG;AACH,wBAAsB,cAAc,CACnC,OAAO,EAAE,qBAAqB,EAC9B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAC7B,OAAO,CAAC,qBAAqB,CAAC,CA8DhC"}

package/out/tools/jsonl-to-parquet.js

@@ -3,129 +3,86 @@
  * @license AGPL-3.0
  * @author Teffen Ellis, et al.
  *
- *   Convert a JSONL of LabeledRow objects to a Parquet slice matching the v0.5.0 schema.
+ * Convert a jsonl of LabeledRow objects to a Parquet file matching the schema the corpus writes.
  *
- *   Ported faithfully from scripts/jsonl-to-parquet.py. The Python original wrote Parquet through
- *   PyArrow; this writes it through DuckDB (`@duckdb/node-api`) — `read_json` with an EXPLICIT
- *   `columns` type map projects the validated rows to the v0.5.0 schema, then `COPY … TO … (FORMAT
- *   PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE …)` emits the slice. DuckDB reproduces the exact
- *   logical schema PyArrow did — `VARCHAR` (UTF8) scalars, `VARCHAR[]` (LIST<UTF8>) for the string
- *   arrays, and `INTEGER[]` (LIST<INT32>) for the span offsets — in the exact column order below.
- *   Verified field-for-field against the PyArrow original (same column order, same logical types,
- *   same `list<element: …>` child naming, same values), so a PyArrow reader sees an identical
- *   table. The trainer in any case reads slices by column name (`pq.read_table(...).to_pylist()`),
- *   which is blind to physical layout. INT32 matches the corpus's native TS writer
- *   (`@mailwoman/corpus` `LABELED_ROW_SCHEMA`), which already writes the base slices this overlay
- *   rides alongside.
+ * DuckDB reproduces the PyArrow file's logical schema and column order. A PyArrow reader therefore sees an identical table.
+ * The trainer reads Parquet by column name, so physical layout does not affect it.
  *
- *   Schema: raw, tokens, labels, span_starts, span_ends, span_tags, country, locale, source,
- *   source_id, corpus_version, license, synth_method, synth_base_id.
+ * Schema: `PARQUET_COLUMNS` from `#parquet/schema`, the same list the native writer uses.
  *
- *   The span triple (#519, v0.5.0 char-offset labels) is REQUIRED on every row: `alignRow` emits it
- *   on every labeled row, so a row arriving without it came from a producer that hasn't migrated —
- *   writing it would silently drop the v0.5.0 labels from the slice. Loud failure, naming the row
- *   number, instead.
- *
- *   Usage: mailwoman dev jsonl-to-parquet --input /tmp/po-box-labeled.jsonl --output
- *   /tmp/part-po-box.parquet
+ * Every row must contain the span triple. A row without it came from a producer that has not migrated.
+ * The writer would silently drop the character-offset labels if it wrote that row. Fail loudly and report the row number.
  */
+import { delimitedSource } from "@mailwoman/core/fs/delimited";
 import { openWriteStream } from "@mailwoman/core/fs/streams";
 import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
-import { parseJSONStrict } from "@mailwoman/core/json";
-import { join } from "path-ts";
+import { parseJSONStrict, stringifyJSON } from "@mailwoman/core/json";
 import { TextSpliterator } from "spliterator";
-import { connectDuckDB, escapeSQLString } from "#utils/parquet";
-const REQUIRED_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb";
+import { PARQUET_COLUMNS, PARQUET_COLUMN_TYPES } from "#parquet/schema";
 /**
- * The v0.5.0 DuckDB type for each column, in {@link REQUIRED_COLUMNS} order.
+ * The columns this converter writes and the DuckDB type for each column.
  *
- * Mirrors the PyArrow schema the Python original declared: `pa.string()` → `VARCHAR`, `pa.list_(pa.string())` →
- * `VARCHAR[]`, `pa.list_(pa.int32())` → `INTEGER[]`. The span offsets are INT32 (#519): parallel arrays over `raw`
- * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping); `raw` is a short address string, so INT32
- * round-trips as a plain integer where INT64 would surface as bigint.
+ * Both come from `#parquet/schema`, the one definition the native writer,
+ * the reader and the manifest share.
+ *
+ * The span offsets are INT32: parallel arrays over `raw`
+ * (UTF-16 code units, `[start, end)` exclusive-end, sorted, non-overlapping).
+ * `raw` is a short address string, so INT32 round-trips as a plain integer
+ * where INT64 would surface as bigint.
  */
-const COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
+const REQUIRED_COLUMNS = PARQUET_COLUMNS;
+const COLUMN_TYPES = PARQUET_COLUMN_TYPES;
+const SPAN_COLUMNS = ["span_starts", "span_ends", "span_tags"];
 /**
- * Enforce the #519 span contract per row: all three present, parallel lengths.
+ * Enforce the span interface per row: all three present, parallel lengths.
  *
- * A row with span_starts but no span_tags is a corrupt row — never a silent fallback.
+ * A row with span_starts but no span_tags is a corrupt row, never a silent fallback.
  */
 function assertSpanTriple(row, lineNo) {
     const present = SPAN_COLUMNS.filter((c) => row[c] != null);
     if (present.length !== SPAN_COLUMNS.length) {
         const missing = SPAN_COLUMNS.filter((c) => row[c] == null);
         throw new Error(`line ${lineNo}: row is missing the char-offset span triple (#519): ` +
-            `missing ${JSON.stringify(missing)} (source_id=${JSON.stringify(row.source_id ?? null)}). Every parquet-bound row ` +
-            "must carry span_starts/span_ends/span_tags; re-emit this slice through alignRow.");
+            `missing ${stringifyJSON(missing)} (source_id=${stringifyJSON(row.source_id ?? null)}). Every parquet-bound row ` +
+            "must carry span_starts/span_ends/span_tags; re-emit this file through alignRow.");
     }
     const n = row.span_starts.length;
     if (row.span_ends.length !== n || row.span_tags.length !== n) {
         throw new Error(`line ${lineNo}: span triple arrays are not parallel — ` +
             `starts=${row.span_starts.length} ends=${row.span_ends.length} ` +
-            `tags=${row.span_tags.length} (source_id=${JSON.stringify(row.source_id ?? null)})`);
+            `tags=${row.span_tags.length} (source_id=${stringifyJSON(row.source_id ?? null)})`);
     }
 }
 /**
- * Convert a labeled-row JSONL to a v0.5.0-schema Parquet slice.
+ * Convert a labeled-row jsonl to a v0.5.0-schema Parquet file.
  */
 export async function jsonlToParquet(options, report) {
     const rowGroupSize = options.rowGroupSize ?? 50_000;
     if (!Number.isInteger(rowGroupSize) || rowGroupSize <= 0) {
-        throw new Error(`rowGroupSize must be a positive integer (got ${JSON.stringify(rowGroupSize)})`);
+        throw new Error(`rowGroupSize must be a positive integer (got ${stringifyJSON(rowGroupSize)})`);
     }
-    // Stage the validated rows to a temp NDJSON, then let DuckDB type + write them. Streaming keeps
-    // memory O(1) on the Node side (the Python original buffered every column into memory first). The
-    // staging directory owns the write stream, so it is closed before the directory is removed — and a
-    // mid-stream span-triple failure leaves no orphan.
+    // The Node-side writer uses O(1) memory by streaming rows.
+    // The staging directory owns the write stream, so it is closed before the directory is removed.
+    // A mid-stream span-triple failure leaves no orphan.
     await using staging = await temporaryDirectory("mw-jsonl-to-parquet-");
-    const stagePath = join(staging.path, "rows.ndjson");
+    const stagePath = staging.path("rows.ndjson");
     const stage = staging.use(openWriteStream(stagePath, { encoding: "utf8" }));
     let rows = 0;
     let lineNo = 0;
-    // TextSpliterator, not JSONSpliterator: the staging write below streams the RAW line bytes to
-    // DuckDB verbatim (the parse here only validates), so a re-serialized JSONSpliterator row would
-    // defeat the point. CRLF is handled by the existing `rawLine.trim()` (strips a trailing \r),
-    // same as readline's crlfDelay:Infinity did.
-    for await (const rawLine of TextSpliterator.fromAsync(options.input)) {
+    // TextSpliterator rather than JSONSpliterator: the staging write below streams
+    // the RAW line bytes to DuckDB verbatim (the parse here only validates),
+    // so a re-serialized JSONSpliterator row would defeat the point.
+    // The reader trims each line (a trailing \r included); `skipEmpty: false` keeps
+    // blank lines so `lineNo` matches the file.
+    for await (const line of TextSpliterator.fromAsync(delimitedSource(options.input), { skipEmpty: false })) {
         lineNo++;
-        const line = rawLine.trim();
         if (!line)
             continue;
         const row = parseJSONStrict(line);
         assertSpanTriple(row, lineNo);
-        // Write the validated line verbatim; DuckDB's `read_json` projects to the explicit `columns`
-        // map below (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
+        // DuckDB's `read_json` projects to the explicit `columns` map below
+        // (extra keys dropped, absent keys → NULL — matching the Python `row.get(c)`).
         stage.write(line + "\n");
         rows++;
     }
@@ -135,16 +92,17 @@ export async function jsonlToParquet(options, report) {
     report?.(`Read ${rows} rows from ${options.input}`);
     const columnsLiteral = "{" + REQUIRED_COLUMNS.map((c) => `'${c}': '${COLUMN_TYPES[c]}'`).join(", ") + "}";
     const selectList = REQUIRED_COLUMNS.join(", ");
-    const db = await connectDuckDB();
-    // Row order is required: the overlay-manifest assembler records first/last source_id from
-    // slice order. `preserve_insertion_order` (DuckDB default) keeps output order = input order.
+    const output = options.output.toString();
+    using db = await openDuckDB();
+    // Row order is required: the overlay-manifest assembler records first/last source_id from file order.
+    // `preserve_insertion_order` (DuckDB default) keeps output order = input order.
     await db.run("SET preserve_insertion_order=true");
-    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+    await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath.toString())}', ` +
         `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-        `TO '${escapeSQLString(options.output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
-    const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(options.output)}')`);
+        `TO '${escapeSQLString(output)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${rowGroupSize})`);
+    const counted = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(output)}')`);
     const written = Number(counted.getRowObjects()[0].n);
-    report?.(`Wrote ${written} rows to ${options.output}`);
-    return { read: rows, written, outPath: options.output };
+    report?.(`Wrote ${written} rows to ${output}`);
+    return { read: rows, written, outPath: output };
 }
 //# sourceMappingURL=jsonl-to-parquet.js.map

package/out/tools/jsonl-to-parquet.js.map

@@ -1 +1 @@
-{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAA;AACtD,OAAO,EAAE,IAAI,EAAE,MAAM,SAAS,CAAA;AAC9B,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAA;AAE/D,MAAM,gBAAgB,GAAG;IACxB,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AAEvE;;;;;;;GAOG;AACH,MAAM,YAAY,GAAsD;IACvE,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AA6BD;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACnH,kFAAkF,CACnF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CACnG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IACjG,CAAC;IAED,gGAAgG;IAChG,kGAAkG;IAClG,mGAAmG;IACnG,mDAAmD;IACnD,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;IACnD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,8FAA8F;IAC9F,gGAAgG;IAChG,6FAA6F;IAC7F,6CAA6C;IAC7C,IAAI,KAAK,EAAE,MAAM,OAAO,IAAI,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACtE,MAAM,EAAE,CAAA;QACR,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAA;QAE3B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,6FAA6F;QAC7F,yFAAyF;QACzF,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;QAExB,IAAI,EAAE,CAAA;IACP,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9C,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,0FAA0F;IAC1F,6FAA6F;IAC7F,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;QAC5E,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CAC/G,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IACtH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAEtD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,MAAM,EAAE,CAAA;AACxD,CAAC"}
+{"version":3,"file":"jsonl-to-parquet.js","sourceRoot":"","sources":["../../lib/tools/jsonl-to-parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,8BAA8B,CAAA;AAC9D,OAAO,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAC5D,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAA;AAErE,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAE7C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC7D,OAAO,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AAEvE;;;;;;;;;;GAUG;AACH,MAAM,gBAAgB,GAAG,eAAe,CAAA;AACxC,MAAM,YAAY,GAAG,oBAAoB,CAAA;AAEzC,MAAM,YAAY,GAAG,CAAC,aAAa,EAAE,WAAW,EAAE,WAAW,CAAU,CAAA;AA+BvE;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,GAA4B,EAAE,MAAc;IACrE,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;IAE1D,IAAI,OAAO,CAAC,MAAM,KAAK,YAAY,CAAC,MAAM,EAAE,CAAC;QAC5C,MAAM,OAAO,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAA;QAC1D,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,uDAAuD;YACpE,WAAW,aAAa,CAAC,OAAO,CAAC,eAAe,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,6BAA6B;YACjH,iFAAiF,CAClF,CAAA;IACF,CAAC;IAED,MAAM,CAAC,GAAI,GAAG,CAAC,WAAyB,CAAC,MAAM,CAAA;IAE/C,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,IAAK,GAAG,CAAC,SAAuB,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5F,MAAM,IAAI,KAAK,CACd,QAAQ,MAAM,0CAA0C;YACvD,UAAW,GAAG,CAAC,WAAyB,CAAC,MAAM,SAAU,GAAG,CAAC,SAAuB,CAAC,MAAM,GAAG;YAC9F,QAAS,GAAG,CAAC,SAAuB,CAAC,MAAM,eAAe,aAAa,CAAC,GAAG,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,CAClG,CAAA;IACF,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,OAA8B,EAC9B,MAA+B;IAE/B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,MAAM,CAAA;IAEnD,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,IAAI,YAAY,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,gDAAgD,aAAa,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;IAChG,CAAC;IAED,2DAA2D;IAC3D,gGAAgG;IAChG,qDAAqD;IACrD,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,sBAAsB,CAAC,CAAA;IACtE,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC7C,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,EAAE,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC,CAAA;IAE3E,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,IAAI,MAAM,GAAG,CAAC,CAAA;IAEd,+EAA+E;IAC/E,yEAAyE;IACzE,iEAAiE;IACjE,gFAAgF;IAChF,4CAA4C;IAC5C,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,eAAe,CAAC,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QAC1G,MAAM,EAAE,CAAA;QAER,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,GAAG,GAAG,eAAe,CAA0B,IAAI,CAAC,CAAA;QAC1D,gBAAgB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC7B,oEAAoE;QACpE,+EAA+E;QAC/E,KAAK,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,CAAA;QAExB,IAAI,EAAE,CAAA;IACP,CAAC;IAED,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,KAAK,CAAC,GAAG,CAAC,CAAC,GAAkB,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IACnE,CAAC,CAAC,CAAA;IAEF,MAAM,EAAE,CAAC,QAAQ,IAAI,cAAc,OAAO,CAAC,KAAK,EAAE,CAAC,CAAA;IAEnD,MAAM,cAAc,GAAG,GAAG,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IACzG,MAAM,UAAU,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9C,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAA;IACxC,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,sGAAsG;IACtG,gFAAgF;IAChF,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;IAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,KAAK;QACvF,aAAa,cAAc,mCAAmC;QAC9D,OAAO,eAAe,CAAC,MAAM,CAAC,yDAAyD,YAAY,GAAG,CACvG,CAAA;IAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC9G,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,CAAA;IACrD,MAAM,EAAE,CAAC,SAAS,OAAO,YAAY,MAAM,EAAE,CAAC,CAAA;IAE9C,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;AAChD,CAAC"}

package/out/tools/lint/recipe-output/vocab.d.ts

@@ -0,0 +1,77 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Base-consistency lint: a synthetic recipe output must not label a token a tag the base dominantly labels
+ * something else, or training gets conflicting gradients on the same token and the minority (the recipe output)
+ * loses. This reads a recipe output's own (token -> tag) and checks each token against the base.
+ *
+ * The check is country-scoped and full-count, because a token's correct tag is country-specific: "Paris" is a
+ * locality in FR data and a street in US "Paris Ave". A cross-country aggregate mis-judges any country-specific
+ * token. A small sample is street-biased because street sources dwarf locality sources. Each
+ * recipe-output token's base tag is therefore tallied scoped to the country the output uses it in, over a large
+ * scan. Pure-numeric tokens are excluded because house_number and postcode are context-determined. An
+ * affix-split flag (recipe output street_suffix/_prefix against base "street") is expected, because the loader's
+ * affix-relabel handles it.
+ */
+/**
+ * Options for {@linkcode lintRecipeVocab}.
+ */
+export interface LintRecipeVocabOptions {
+    /**
+     * The recipe output parquet to lint.
+     */
+    recipeOutputPath: string;
+    /**
+     * Base corpus version.
+     * Default `v0.5.0`.
+     */
+    baseVersion?: string;
+    /**
+     * Base corpus root.
+     * Default `$MAILWOMAN_DATA_ROOT/corpus/versioned`.
+     */
+    baseRoot?: string;
+    /**
+     * Base-majority confidence floor for a contradiction.
+     * Default 0.7.
+     */
+    threshold?: number;
+    /**
+     * Minimum base support to judge a token.
+     * Default 50.
+     */
+    minCount?: number;
+    /**
+     * Fraction of base parts to scan, a proportional per-source sample below 1.0.
+     * Default 1.0.
+     */
+    fraction?: number;
+}
+/**
+ * One contradiction row: token, recipe-output tag, base tag, base fraction, base total.
+ */
+export type VocabRow = [token: string, outputTag: string, baseTag: string, baseFrac: number, baseTotal: number];
+/**
+ * Findings summary returned by {@linkcode lintRecipeVocab}.
+ */
+export interface LintRecipeVocabSummary {
+    /**
+     * Real contradictions — the command exits 1 when nonzero.
+     */
+    errors: number;
+    /**
+     * Affix-split rows (expected — the loader's affix-relabel handles them).
+     */
+    warnings: number;
+    findings: {
+        contradictions: VocabRow[];
+        affixSplits: VocabRow[];
+    };
+}
+/**
+ * Lint a synthetic recipe output's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export declare function lintRecipeVocab(options: LintRecipeVocabOptions): Promise<LintRecipeVocabSummary>;
+//# sourceMappingURL=vocab.d.ts.map

package/out/tools/lint/recipe-output/vocab.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.d.ts","sourceRoot":"","sources":["../../../../lib/tools/lint/recipe-output/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AA+IH;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,gBAAgB,EAAE,MAAM,CAAA;IACxB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;GAEG;AACH,MAAM,MAAM,QAAQ,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;AAE/G;;GAEG;AACH,MAAM,WAAW,sBAAsB;IACtC;;OAEG;IACH,MAAM,EAAE,MAAM,CAAA;IACd;;OAEG;IACH,QAAQ,EAAE,MAAM,CAAA;IAChB,QAAQ,EAAE;QAAE,cAAc,EAAE,QAAQ,EAAE,CAAC;QAAC,WAAW,EAAE,QAAQ,EAAE,CAAA;KAAE,CAAA;CACjE;AAED;;GAEG;AACH,wBAAsB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CAoJtG"}

package/out/tools/lint/recipe-output/vocab.js

@@ -0,0 +1,239 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ * Base-consistency lint: a synthetic recipe output must not label a token a tag the base dominantly labels
+ * something else, or training gets conflicting gradients on the same token and the minority (the recipe output)
+ * loses. This reads a recipe output's own (token -> tag) and checks each token against the base.
+ *
+ * The check is country-scoped and full-count, because a token's correct tag is country-specific: "Paris" is a
+ * locality in FR data and a street in US "Paris Ave". A cross-country aggregate mis-judges any country-specific
+ * token. A small sample is street-biased because street sources dwarf locality sources. Each
+ * recipe-output token's base tag is therefore tallied scoped to the country the output uses it in, over a large
+ * scan. Pure-numeric tokens are excluded because house_number and postcode are context-determined. An
+ * affix-split flag (recipe output street_suffix/_prefix against base "street") is expected, because the loader's
+ * affix-relabel handles it.
+ */
+import { dataRootPath } from "@mailwoman/core/data-root";
+import { tryParsingJSON } from "@mailwoman/core/json";
+import { pyRound } from "@mailwoman/core/numeric";
+import { PathBuilder } from "path-ts";
+import { Globerator } from "spliterator/node/fs";
+import { openDuckDB } from "#parquet/duckdb";
+/**
+ * Strip a BIO prefix ("B-"/"I-") off a label, matching the Python `strip_bio`.
+ */
+function stripBIO(label) {
+    const head = label.slice(0, 2);
+    return head === "B-" || head === "I-" ? label.slice(2) : label;
+}
+/**
+ * Python `str.isdigit()`: non-empty and every character a Unicode digit.
+ *
+ * Pure-numeric tokens (house_number and postcode) are context-determined
+ * rather than lexical vocab, so they are excluded.
+ * `\p{Nd}` covers the decimal digits these address corpora actually contain.
+ */
+function isDigit(token) {
+    return token.length > 0 && /^\p{Nd}+$/u.test(token);
+}
+/**
+ * Format a fraction as a whole-percent string the way Python's `:.0%` does, e.g. 0.73 -> "73%".
+ */
+function pct(frac) {
+    return `${pyRound(frac * 100)}%`;
+}
+/**
+ * Format a float the way a Python f-string renders it: integer-valued floats keep one
+ * decimal (1.0 -> "1.0"), everything else is its shortest decimal (0.5 -> "0.5").
+ *
+ * Used for the `fraction` echo so the banner matches the Python print.
+ */
+function formatPyFloat(n) {
+    return Number.isInteger(n) ? n.toFixed(1) : String(n);
+}
+/**
+ * Left-justify to a minimum width with spaces, matching Python's `{value:N}` string field.
+ */
+function pad(value, width) {
+    return value.padEnd(width);
+}
+/**
+ * The dominant tag of a counter as (tag, total, fraction).
+ *
+ * An empty counter yields ("", 0, 0.0), and ties go to the first-inserted tag,
+ * mirroring `Counter.most_common(1)`.
+ */
+function dominant(counter) {
+    let total = 0;
+    let bestTag = "";
+    let bestCount = -1;
+    for (const [tag, count] of counter) {
+        total += count;
+        if (count > bestCount) {
+            bestCount = count;
+            bestTag = tag;
+        }
+    }
+    if (total === 0)
+        return ["", 0, 0];
+    return [bestTag, total, bestCount / total];
+}
+/**
+ * Bump a (key -> count) tally, creating the inner counter on first sight.
+ */
+function bump(table, key, sub) {
+    let counter = table.get(key);
+    if (!counter) {
+        counter = new Map();
+        table.set(key, counter);
+    }
+    counter.set(sub, (counter.get(sub) ?? 0) + 1);
+}
+/**
+ * Read a corpus parquet into rows, projecting only tokens, labels and country.
+ *
+ * DuckDB `to_json` returns list columns as JSON text.
+ * Gazetteer builders use the same technique for nested columns.
+ * This function parses the text back to string arrays.
+ */
+async function readRows(con, path) {
+    const result = await con.runAndReadAll(`SELECT to_json(tokens) AS tokens, to_json(labels) AS labels, country FROM read_parquet('${path}')`);
+    const raw = result.getRowObjects();
+    const rows = [];
+    for (const r of raw) {
+        const tokens = tryParsingJSON(String(r.tokens));
+        const labels = tryParsingJSON(String(r.labels));
+        if (!Array.isArray(tokens) || !Array.isArray(labels))
+            continue;
+        rows.push({
+            tokens: tokens,
+            labels: labels,
+            country: r.country == null ? null : String(r.country),
+        });
+    }
+    return rows;
+}
+/**
+ * Read just the first row's `source` value, used to group base parts for a proportional sample.
+ */
+async function readSource(con, path) {
+    const result = await con.runAndReadAll(`SELECT source FROM read_parquet('${path}') LIMIT 1`);
+    const rows = result.getRowObjects();
+    return rows.length ? String(rows[0].source) : "";
+}
+/**
+ * Lint a synthetic recipe output's (token → tag) vocabulary against the base corpus, country-scoped.
+ */
+export async function lintRecipeVocab(options) {
+    const baseVersion = options.baseVersion ?? "v0.5.0";
+    const baseRoot = PathBuilder.from(options.baseRoot ?? dataRootPath("corpus", "versioned"));
+    const threshold = options.threshold ?? 0.7;
+    const minCount = options.minCount ?? 50;
+    const fraction = options.fraction ?? 1;
+    using con = await openDuckDB();
+    const outputRows = await readRows(con, options.recipeOutputPath);
+    const outputTags = new Map();
+    const outputCountries = new Map();
+    for (const { tokens, labels, country } of outputRows) {
+        const n = Math.min(tokens.length, labels.length);
+        for (let i = 0; i < n; i++) {
+            const w = tokens[i];
+            const l = labels[i];
+            if (isDigit(w))
+                continue;
+            bump(outputTags, w, stripBIO(l));
+            let set = outputCountries.get(w);
+            if (!set) {
+                set = new Set();
+                outputCountries.set(w, set);
+            }
+            set.add(country);
+        }
+    }
+    const outputVocab = new Set(outputTags.keys());
+    console.log(`recipe output: ${outputRows.length} rows, ${outputVocab.size} unique tokens`);
+    const trainDir = baseRoot(baseVersion, `corpus-${baseVersion}`, "train");
+    let parts = (await Globerator.from("*.parquet", {
+        cwd: trainDir,
+        absolute: true,
+    }).toArray()).toSorted();
+    if (!parts.length) {
+        throw new Error("no base parts found");
+    }
+    if (fraction < 1) {
+        const bysrc = new Map();
+        for (const p of parts) {
+            const src = await readSource(con, p);
+            let list = bysrc.get(src);
+            if (!list) {
+                list = [];
+                bysrc.set(src, list);
+            }
+            list.push(p);
+        }
+        const sampled = [];
+        for (const ps of bysrc.values()) {
+            const take = Math.max(2, pyRound(ps.length * fraction));
+            for (const p of ps.slice(0, take)) {
+                sampled.push(p);
+            }
+        }
+        parts = sampled;
+    }
+    console.log(`base ${baseVersion}: scanning ${parts.length} parts (fraction=${formatPyFloat(fraction)}), COUNTRY-scoped`);
+    const baseTags = new Map();
+    for (let i = 0; i < parts.length; i++) {
+        const rows = await readRows(con, parts[i]);
+        for (const { tokens, labels, country } of rows) {
+            const n = Math.min(tokens.length, labels.length);
+            for (let j = 0; j < n; j++) {
+                const w = tokens[j];
+                if (outputVocab.has(w) && outputCountries.get(w).has(country)) {
+                    bump(baseTags, w, stripBIO(labels[j]));
+                }
+            }
+        }
+        if ((i + 1) % 100 === 0) {
+            console.log(`  ...${i + 1}/${parts.length} parts`);
+        }
+    }
+    const flagged = [];
+    const affix = [];
+    for (const w of outputVocab) {
+        const [sTag] = dominant(outputTags.get(w));
+        const [bTag, bTotal, bFrac] = dominant(baseTags.get(w) ?? new Map());
+        if (bTotal < minCount || !bTag || bTag === sTag || bFrac < threshold)
+            continue;
+        const row = [w, sTag, bTag, bFrac, bTotal];
+        if ((sTag === "street_suffix" || sTag === "street_prefix") && bTag === "street") {
+            affix.push(row);
+        }
+        else {
+            flagged.push(row);
+        }
+    }
+    const sections = [
+        ["CONTRADICTION", flagged],
+        ["affix-split (EXPECTED — affix-relabel handles)", affix],
+    ];
+    for (const [label, rows] of sections) {
+        if (!rows.length)
+            continue;
+        rows.sort((a, b) => b[4] - a[4] || b[3] - a[3]);
+        console.log(`\n${label.startsWith("CONTRA") ? "⚠️ " : "· "}${rows.length} ${label}:`);
+        for (const [w, sTag, bTag, bFrac, bTotal] of rows) {
+            console.log(`  ${pad(w, 18)} output=${pad(sTag, 14)} base=${bTag} (${pct(bFrac)}, n=${bTotal})`);
+        }
+    }
+    if (!flagged.length) {
+        console.log(`\n✅ NO real contradictions (country-scoped, threshold ${pct(threshold)}, support ${minCount}) — recipe output base-consistent`);
+    }
+    return {
+        errors: flagged.length,
+        warnings: affix.length,
+        findings: { contradictions: flagged, affixSplits: affix },
+    };
+}
+//# sourceMappingURL=vocab.js.map

package/out/tools/lint/recipe-output/vocab.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"vocab.js","sourceRoot":"","sources":["../../../../lib/tools/lint/recipe-output/vocab.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAA;AACxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAA;AACrD,OAAO,EAAE,OAAO,EAAE,MAAM,yBAAyB,CAAA;AACjD,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAA;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAA;AAEhD,OAAO,EAAyB,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAWnE;;GAEG;AACH,SAAS,QAAQ,CAAC,KAAa;IAC9B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;IAE9B,OAAO,IAAI,KAAK,IAAI,IAAI,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAA;AAC/D,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,KAAa;IAC7B,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;AACpD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,IAAY;IACxB,OAAO,GAAG,OAAO,CAAC,IAAI,GAAG,GAAG,CAAC,GAAG,CAAA;AACjC,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,CAAS;IAC/B,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;AACtD,CAAC;AAED;;GAEG;AACH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAa;IACxC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AAC3B,CAAC;AAED;;;;;GAKG;AACH,SAAS,QAAQ,CAAC,OAA4B;IAC7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,IAAI,OAAO,GAAG,EAAE,CAAA;IAChB,IAAI,SAAS,GAAG,CAAC,CAAC,CAAA;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;QACpC,KAAK,IAAI,KAAK,CAAA;QAEd,IAAI,KAAK,GAAG,SAAS,EAAE,CAAC;YACvB,SAAS,GAAG,KAAK,CAAA;YACjB,OAAO,GAAG,GAAG,CAAA;QACd,CAAC;IACF,CAAC;IAED,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAA;IAElC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,SAAS,GAAG,KAAK,CAAC,CAAA;AAC3C,CAAC;AAED;;GAEG;AACH,SAAS,IAAI,CAAC,KAAuC,EAAE,GAAW,EAAE,GAAW;IAC9E,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;;;;GAMG;AACH,KAAK,UAAU,QAAQ,CAAC,GAAqB,EAAE,IAAY;IAC1D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CACrC,2FAA2F,IAAI,IAAI,CACnG,CAAA;IAED,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,EAAmE,CAAA;IACnG,MAAM,IAAI,GAAgB,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;QACrB,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAC/C,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;QAE/C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,SAAQ;QAE9D,IAAI,CAAC,IAAI,CAAC;YACT,MAAM,EAAE,MAAkB;YAC1B,MAAM,EAAE,MAAkB;YAC1B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SACrD,CAAC,CAAA;IACH,CAAC;IAED,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,GAAqB,EAAE,IAAY;IAC5D,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,aAAa,CAAC,oCAAoC,IAAI,YAAY,CAAC,CAAA;IAC5F,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAAgC,CAAA;IAEjE,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AAClD,CAAC;AAyDD;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,OAA+B;IACpE,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,QAAQ,CAAA;IACnD,MAAM,QAAQ,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,YAAY,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC,CAAA;IAC1F,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAA;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAA;IACvC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;IAEtC,MAAM,GAAG,GAAG,MAAM,UAAU,EAAE,CAAA;IAE9B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAA;IAChE,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAA;IACzD,MAAM,eAAe,GAAG,IAAI,GAAG,EAA8B,CAAA;IAE7D,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,UAAU,EAAE,CAAC;QACtD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;QAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YACpB,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;YAEpB,IAAI,OAAO,CAAC,CAAC,CAAC;gBAAE,SAAQ;YACxB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAA;YAChC,IAAI,GAAG,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,CAAA;YAEhC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACV,GAAG,GAAG,IAAI,GAAG,EAAE,CAAA;gBACf,eAAe,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;YAC5B,CAAC;YAED,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;QACjB,CAAC;IACF,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAA;IAE9C,OAAO,CAAC,GAAG,CAAC,kBAAkB,UAAU,CAAC,MAAM,UAAU,WAAW,CAAC,IAAI,gBAAgB,CAAC,CAAA;IAE1F,MAAM,QAAQ,GAAG,QAAQ,CAAC,WAAW,EAAE,UAAU,WAAW,EAAE,EAAE,OAAO,CAAC,CAAA;IAExE,IAAI,KAAK,GAAG,CACX,MAAM,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE;QAClC,GAAG,EAAE,QAAQ;QACb,QAAQ,EAAE,IAAI;KACd,CAAC,CAAC,OAAO,EAAE,CACZ,CAAC,QAAQ,EAAE,CAAA;IAEZ,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QACnB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAA;IACvC,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAA;QAEzC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE,CAAC,CAAC,CAAA;YACpC,IAAI,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAEzB,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,IAAI,GAAG,EAAE,CAAA;gBACT,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;YACrB,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACb,CAAC;QAED,MAAM,OAAO,GAAa,EAAE,CAAA;QAE5B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACjC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,MAAM,GAAG,QAAQ,CAAC,CAAC,CAAA;YAEvD,KAAK,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;QACF,CAAC;QAED,KAAK,GAAG,OAAO,CAAA;IAChB,CAAC;IAED,OAAO,CAAC,GAAG,CACV,QAAQ,WAAW,cAAc,KAAK,CAAC,MAAM,oBAAoB,aAAa,CAAC,QAAQ,CAAC,mBAAmB,CAC3G,CAAA;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAA;IAEvD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACvC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAE,CAAC,CAAA;QAE3C,KAAK,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC;YAChD,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;YAEhD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;gBAEpB,IAAI,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;oBAChE,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAA;gBACxC,CAAC;YACF,CAAC;QACF,CAAC;QAED,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,QAAQ,CAAC,CAAA;QACnD,CAAC;IACF,CAAC;IAED,MAAM,OAAO,GAAe,EAAE,CAAA;IAC9B,MAAM,KAAK,GAAe,EAAE,CAAA;IAE5B,KAAK,MAAM,CAAC,IAAI,WAAW,EAAE,CAAC;QAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAE,CAAC,CAAA;QAC3C,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,GAAG,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAA;QAEpE,IAAI,MAAM,GAAG,QAAQ,IAAI,CAAC,IAAI,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,GAAG,SAAS;YAAE,SAAQ;QAC9E,MAAM,GAAG,GAAa,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,CAAA;QAEpD,IAAI,CAAC,IAAI,KAAK,eAAe,IAAI,IAAI,KAAK,eAAe,CAAC,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;YACjF,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAChB,CAAC;aAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAClB,CAAC;IACF,CAAC;IAED,MAAM,QAAQ,GAAgC;QAC7C,CAAC,eAAe,EAAE,OAAO,CAAC;QAC1B,CAAC,gDAAgD,EAAE,KAAK,CAAC;KACzD,CAAA;IAED,KAAK,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,QAAQ,EAAE,CAAC;QACtC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,SAAQ;QAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAE/C,OAAO,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,CAAA;QAErF,KAAK,MAAM,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACnD,OAAO,CAAC,GAAG,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,SAAS,IAAI,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,MAAM,GAAG,CAAC,CAAA;QACjG,CAAC;IACF,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;QACrB,OAAO,CAAC,GAAG,CACV,yDAAyD,GAAG,CAAC,SAAS,CAAC,aAAa,QAAQ,mCAAmC,CAC/H,CAAA;IACF,CAAC;IAED,OAAO;QACN,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE,cAAc,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE;KACzD,CAAA;AACF,CAAC"}