@mailwoman/corpus 9.1.0 → 9.3.0

package/out/utils/parquet.d.ts

@@ -0,0 +1,178 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Final output sliceer for the corpus pipeline.
+ *
+ *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
+ *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
+ *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
+ *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
+ *
+ *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
+ *   which is the standard ML-corpus default (and PyArrow's default). Documented in
+ *   `DECISIONS.md`.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *   MANIFEST.json
+ *   train/
+ *     part-0000.parquet
+ *     part-0001.parquet
+ *     ...
+ *   val/
+ *     part-0000.parquet
+ *   test/
+ *     part-0000.parquet
+ * ```
+ *
+ *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
+ *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
+ *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
+ *   close — cheap relative to writing it).
+ */
+import { type PathBuilderLike } from "path-ts";
+import type { LabeledRow } from "#types";
+import type { SplitName } from "#utils/split";
+/**
+ * Row groups are written at this cadence within a slice.
+ */
+export declare const ROW_GROUP_SIZE = 50000;
+/**
+ * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
+ */
+export declare function escapeSQLString(value: string): string;
+/**
+ * An open DuckDB connection, re-exported so consumers of {@link connectDuckDB} can name the type without their own
+ * static dependency on the optional peer.
+ */
+export type { DuckDBConnection } from "@duckdb/node-api";
+/**
+ * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
+ * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
+ */
+export declare function connectDuckDB(): Promise<import("@duckdb/node-api").DuckDBConnection>;
+/**
+ * Snappy is the codec selected for corpus slices.
+ */
+export declare const SLICE_COMPRESSION: "SNAPPY";
+export interface ParquetFieldDefinition {
+    type: "UTF8" | "INT32";
+    compression: typeof SLICE_COMPRESSION;
+    repeated?: boolean;
+    optional?: boolean;
+}
+export type ParquetSchemaDefinition<T> = Record<Extract<keyof T, string>, ParquetFieldDefinition>;
+/**
+ * A single Parquet row shape. The index signature allows callers to carry source fields before projection.
+ *
+ * Optional fields are represented as null in the Arrow table and read back as null.
+ */
+export interface ParquetRow {
+    raw: string;
+    tokens: readonly string[];
+    labels: readonly string[];
+    span_starts: readonly number[];
+    span_ends: readonly number[];
+    span_tags: readonly string[];
+    country: string;
+    locale?: string | null;
+    source: string;
+    source_id: string;
+    corpus_version: string;
+    license: string;
+    synth_method?: string | null;
+    synth_base_id?: string | null;
+    [key: string]: unknown;
+}
+/**
+ * Column names emitted into every slice. Matches `ParquetRow`.
+ */
+export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "synth_method", "synth_base_id"];
+/**
+ * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
+ * tokens/labels arrays. Compression is per-column SNAPPY.
+ */
+export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
+export declare function writeParquetRows(rows: readonly ParquetRow[], path: string): Promise<void>;
+/**
+ * Stream rows from a local Parquet file in DuckDB-managed chunks.
+ *
+ * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
+ * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
+ */
+export declare function streamParquetRows<T>(path: string, columns?: readonly string[], options?: {
+    limit?: number;
+}): AsyncGenerator<T>;
+/**
+ * Per-slice metadata captured in `MANIFEST.json`.
+ */
+export interface SliceDescriptor {
+    split: SplitName;
+    path: string;
+    format: "parquet";
+    compression: typeof SLICE_COMPRESSION;
+    rows: number;
+    bytes: number;
+    sha256: string;
+    first_source_id: string;
+    last_source_id: string;
+    /**
+     * The slice's corpus source slug, when the writer knows it. `audit.ts` prefers this over inferring the source from
+     * `first_source_id`'s prefix; `writeSlices` itself writes multi-source slices and leaves it unset.
+     */
+    source?: string;
+}
+export interface SliceManifest {
+    corpus_version: string;
+    schema: readonly string[];
+    rows_per_slice: number;
+    row_group_size: number;
+    slices: SliceDescriptor[];
+    counts: Record<SplitName, number>;
+    total_rows: number;
+}
+export interface WriteSlicesOptions {
+    /**
+     * Root output directory; corpus version dir is created beneath.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Corpus version stamped onto rows + into the output directory name.
+     */
+    corpusVersion: string;
+    /**
+     * Max rows per `.parquet` slice. Default 1_000_000 per the Phase 1 plan.
+     */
+    rowsPerSlice?: number;
+}
+/**
+ * Pre-partitioned labeled-row streams, one per split. Callers (`buildCorpus`) decide each row's split inline at align
+ * time via `splitForRow` and route rows to the matching stream, eliminating the prior `Map<source_id, SplitName>` O(n)
+ * lookup table.
+ *
+ * Splits with no rows can be omitted (or passed as an empty iterable); `writeSlices` skips them.
+ */
+export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
+/**
+ * Project a labeled row to the Parquet schema.
+ *
+ * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
+ * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
+ * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
+ */
+export declare function rowToParquet(row: LabeledRow): ParquetRow;
+/**
+ * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
+ * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
+ * Parquet file from disk.
+ *
+ * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
+ * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
+ * for the new wire-up.
+ */
+export declare function writeSlices(perSplit: PerSplitRows, opts: WriteSlicesOptions): Promise<SliceManifest>;
+//# sourceMappingURL=parquet.d.ts.map

package/out/utils/parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../lib/utils/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAUH,OAAO,EAAQ,KAAK,eAAe,EAAE,MAAM,SAAS,CAAA;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAE7C;;GAEG;AACH,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;GAGG;AACH,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAExD;;;GAGG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,OAAO,kBAAkB,EAAE,gBAAgB,CAAC,CAK1F;AAED;;GAEG;AACH,eAAO,MAAM,iBAAiB,EAAG,QAAiB,CAAA;AAElD,MAAM,WAAW,sBAAsB;IAEtC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IACtB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,sBAAsB,CAAC,CAAA;AAEjG;;;;GAIG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC5B,aAAa,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC7B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;GAEG;AACH,eAAO,MAAM,eAAe,yLAelB,CAAA;AAMV;;;GAGG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAmFD,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAa/F;AAYD;;;;;GAKG;AACH,wBAAuB,iBAAiB,CAAC,CAAC,EACzC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,EAC3B,OAAO,CAAC,EAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAA;CAAE,GAC1B,cAAc,CAAC,CAAC,CAAC,CAgCnB;AAyCD;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,iBAAiB,CAAA;IACrC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,aAAa;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,eAAe,EAAE,CAAA;IACzB,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,kBAAkB;IAClC;;OAEG;IACH,SAAS,EAAE,eAAe,CAAA;IAE1B;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAA;CACrB;AAED;;;;;;GAMG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AAEhF;;;;;;GAMG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAqCxD;AAED;;;;;;;;GAQG;AACH,wBAAsB,WAAW,CAAC,QAAQ,EAAE,YAAY,EAAE,IAAI,EAAE,kBAAkB,GAAG,OAAO,CAAC,aAAa,CAAC,CAoH1G"}

package/out/utils/parquet.js

@@ -0,0 +1,371 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Final output sliceer for the corpus pipeline.
+ *
+ *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
+ *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
+ *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
+ *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
+ *
+ *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
+ *   which is the standard ML-corpus default (and PyArrow's default). Documented in
+ *   `DECISIONS.md`.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *   MANIFEST.json
+ *   train/
+ *     part-0000.parquet
+ *     part-0001.parquet
+ *     ...
+ *   val/
+ *     part-0000.parquet
+ *   test/
+ *     part-0000.parquet
+ * ```
+ *
+ *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
+ *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
+ *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
+ *   close — cheap relative to writing it).
+ */
+import { tryStat } from "@mailwoman/core/fs/readers";
+import { openWriteStream } from "@mailwoman/core/fs/streams";
+import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
+import { writeLocalBuffer, writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers";
+import { sha256File } from "@mailwoman/core/hash";
+import { once } from "@mailwoman/core/utils/events";
+import { Field, Int32, List, Table as ArrowTable, tableToIPC, Utf8, vectorFromArray } from "apache-arrow";
+import { Compression, Table as WasmTable, WriterPropertiesBuilder, writeParquet } from "parquet-wasm";
+import { join } from "path-ts";
+/**
+ * Row groups are written at this cadence within a slice.
+ */
+export const ROW_GROUP_SIZE = 50_000;
+/**
+ * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
+ */
+export function escapeSQLString(value) {
+    return value.replaceAll("'", "''");
+}
+/**
+ * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
+ * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
+ */
+export async function connectDuckDB() {
+    const { DuckDBInstance } = await import("@duckdb/node-api");
+    const instance = await DuckDBInstance.create();
+    return await instance.connect();
+}
+/**
+ * Snappy is the codec selected for corpus slices.
+ */
+export const SLICE_COMPRESSION = "SNAPPY";
+/**
+ * Column names emitted into every slice. Matches `ParquetRow`.
+ */
+export const PARQUET_COLUMNS = [
+    "raw",
+    "tokens",
+    "labels",
+    "span_starts",
+    "span_ends",
+    "span_tags",
+    "country",
+    "locale",
+    "source",
+    "source_id",
+    "corpus_version",
+    "license",
+    "synth_method",
+    "synth_base_id",
+];
+/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
+   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
+   and the rule has no way to tell the two apart. */
+/**
+ * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
+ * tokens/labels arrays. Compression is per-column SNAPPY.
+ */
+export const LABELED_ROW_SCHEMA = {
+    raw: { type: "UTF8", compression: SLICE_COMPRESSION },
+    tokens: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
+    labels: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
+    // v0.5.0 char-offset label spans (#519): parallel arrays over `raw` (UTF-16 code units,
+    // [start, end) exclusive-end, sorted, non-overlapping). INT32 — raw is a short address string,
+    // and INT32 round-trips as `number` where parquetjs INT64 would surface bigint.
+    span_starts: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
+    span_ends: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
+    span_tags: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
+    country: { type: "UTF8", compression: SLICE_COMPRESSION },
+    locale: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
+    source: { type: "UTF8", compression: SLICE_COMPRESSION },
+    source_id: { type: "UTF8", compression: SLICE_COMPRESSION },
+    corpus_version: { type: "UTF8", compression: SLICE_COMPRESSION },
+    license: { type: "UTF8", compression: SLICE_COMPRESSION },
+    synth_method: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
+    synth_base_id: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
+};
+const stringListType = new List(new Field("item", new Utf8(), true));
+const int32ListType = new List(new Field("item", new Int32(), true));
+const PARQUET_COLUMN_TYPES = {
+    raw: "VARCHAR",
+    tokens: "VARCHAR[]",
+    labels: "VARCHAR[]",
+    span_starts: "INTEGER[]",
+    span_ends: "INTEGER[]",
+    span_tags: "VARCHAR[]",
+    country: "VARCHAR",
+    locale: "VARCHAR",
+    source: "VARCHAR",
+    source_id: "VARCHAR",
+    corpus_version: "VARCHAR",
+    license: "VARCHAR",
+    synth_method: "VARCHAR",
+    synth_base_id: "VARCHAR",
+};
+function parquetTable(rows) {
+    return new ArrowTable({
+        raw: vectorFromArray(rows.map((row) => row.raw), new Utf8()),
+        tokens: vectorFromArray(rows.map((row) => row.tokens), stringListType),
+        labels: vectorFromArray(rows.map((row) => row.labels), stringListType),
+        span_starts: vectorFromArray(rows.map((row) => row.span_starts), int32ListType),
+        span_ends: vectorFromArray(rows.map((row) => row.span_ends), int32ListType),
+        span_tags: vectorFromArray(rows.map((row) => row.span_tags), stringListType),
+        country: vectorFromArray(rows.map((row) => row.country), new Utf8()),
+        locale: vectorFromArray(rows.map((row) => row.locale ?? null), new Utf8()),
+        source: vectorFromArray(rows.map((row) => row.source), new Utf8()),
+        source_id: vectorFromArray(rows.map((row) => row.source_id), new Utf8()),
+        corpus_version: vectorFromArray(rows.map((row) => row.corpus_version), new Utf8()),
+        license: vectorFromArray(rows.map((row) => row.license), new Utf8()),
+        synth_method: vectorFromArray(rows.map((row) => row.synth_method ?? null), new Utf8()),
+        synth_base_id: vectorFromArray(rows.map((row) => row.synth_base_id ?? null), new Utf8()),
+    });
+}
+export async function writeParquetRows(rows, path) {
+    const arrow = parquetTable(rows);
+    const wasmTable = WasmTable.fromIPCStream(tableToIPC(arrow, "stream"));
+    const properties = new WriterPropertiesBuilder()
+        .setCompression(Compression.SNAPPY)
+        .setCreatedBy("mailwoman")
+        .setMaxRowGroupSize(ROW_GROUP_SIZE)
+        .build();
+    // parquet-wasm serializes key-value metadata through a hash map, whose order is not stable between writes.
+    // Slice identity and provenance live in MANIFEST.json, so omitting file metadata preserves deterministic bytes.
+    await writeLocalBuffer(writeParquet(wasmTable, properties), path);
+}
+function normalizeDuckDBValue(value) {
+    if (value && typeof value === "object" && "items" in value && Array.isArray(value.items)) {
+        return value.items.map(normalizeDuckDBValue);
+    }
+    if (Array.isArray(value))
+        return value.map(normalizeDuckDBValue);
+    return value;
+}
+/**
+ * Stream rows from a local Parquet file in DuckDB-managed chunks.
+ *
+ * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
+ * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
+ */
+export async function* streamParquetRows(path, columns, options) {
+    const db = await connectDuckDB();
+    const projection = columns?.length ? columns.map(escapeSQLIdentifier).join(", ") : "*";
+    const limit = options?.limit;
+    const limitClause = limit === undefined ? "" : ` LIMIT ${validateLimit(limit)}`;
+    const sql = `SELECT ${projection} FROM read_parquet('${escapeSQLString(path)}')${limitClause}`;
+    try {
+        const stream = await db.stream(sql);
+        const columnNames = stream.columnNames();
+        for (let chunk = await stream.fetchChunk(); chunk && chunk.rowCount > 0; chunk = await stream.fetchChunk()) {
+            const rows = chunk.getRowObjects(columnNames);
+            for (const row of rows) {
+                yield Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeDuckDBValue(value)]));
+            }
+        }
+    }
+    catch (error) {
+        if (columns && /Binder Error|Referenced column|does not exist/.test(String(error))) {
+            const missing = String(error).match(/(?:Referenced column|field named) ["']([^"']+)["']/)?.[1] ?? columns.join(", ");
+            throw new Error(`Parquet projection requested column absent from the file schema: ${missing}`, {
+                cause: error,
+            });
+        }
+        throw error;
+    }
+    finally {
+        db.closeSync();
+    }
+}
+function escapeSQLIdentifier(value) {
+    return `"${value.replaceAll('"', '""')}"`;
+}
+function validateLimit(value) {
+    if (!Number.isSafeInteger(value) || value < 0)
+        throw new Error(`Parquet row limit must be a non-negative integer: ${value}`);
+    return value;
+}
+async function writeStagedParquet(stagePath, outputPath) {
+    const db = await connectDuckDB();
+    const columns = [...PARQUET_COLUMNS];
+    const columnsLiteral = "{" + columns.map((column) => `'${column}': '${PARQUET_COLUMN_TYPES[column]}'`).join(", ") + "}";
+    const selectList = columns.map(escapeSQLIdentifier).join(", ");
+    try {
+        await db.run("SET preserve_insertion_order=true");
+        await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
+            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
+            `TO '${escapeSQLString(outputPath)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`);
+    }
+    finally {
+        db.closeSync();
+    }
+}
+async function writeStagedRow(stage, row) {
+    if (!stage.write(JSON.stringify(row) + "\n")) {
+        await once(stage, "drain");
+    }
+}
+/**
+ * Project a labeled row to the Parquet schema.
+ *
+ * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
+ * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
+ * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
+ */
+export function rowToParquet(row) {
+    const { span_starts, span_ends, span_tags } = row;
+    if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
+        throw new Error(`rowToParquet: row is missing the char-offset span triple (#519) — ` +
+            `span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
+            `(source=${row.source}, source_id=${row.source_id}). ` +
+            `Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
+            `producers that emit tokens/labels only have not migrated to the v0.5.0 format.`);
+    }
+    if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
+        throw new Error(`rowToParquet: span triple arrays are not parallel — ` +
+            `starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
+            `(source=${row.source}, source_id=${row.source_id})`);
+    }
+    return {
+        raw: row.raw,
+        tokens: row.tokens,
+        labels: row.labels,
+        span_starts,
+        span_ends,
+        span_tags,
+        country: row.country,
+        locale: row.locale ?? null,
+        source: row.source,
+        source_id: row.source_id,
+        corpus_version: row.corpus_version,
+        license: row.license,
+        synth_method: row.synth?.method ?? null,
+        synth_base_id: row.synth?.base_source_id ?? null,
+    };
+}
+/**
+ * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
+ * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
+ * Parquet file from disk.
+ *
+ * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
+ * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
+ * for the new wire-up.
+ */
+export async function writeSlices(perSplit, opts) {
+    const rowsPerSlice = opts.rowsPerSlice ?? 1_000_000;
+    const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`);
+    await makeDirectories(corpusDir);
+    const slices = [];
+    const counts = { train: 0, val: 0, test: 0 };
+    let totalRows = 0;
+    for (const split of ["train", "val", "test"]) {
+        const rows = perSplit[split];
+        if (!rows)
+            continue;
+        await using staging = await temporaryDirectory(`mailwoman-parquet-${split}-`);
+        let sliceIndex = 0;
+        let path = "";
+        let stagePath = "";
+        let sliceRows = 0;
+        let stage = null;
+        let firstSourceID = "";
+        let lastSourceID = "";
+        const openSlice = async () => {
+            const splitDir = join(corpusDir, split);
+            await makeDirectories(splitDir);
+            path = join(splitDir, `part-${String(sliceIndex).padStart(4, "0")}.parquet`);
+            stagePath = staging.resolve(`part-${String(sliceIndex).padStart(4, "0")}.ndjson`);
+            stage = staging.use(openWriteStream(stagePath));
+            sliceRows = 0;
+            firstSourceID = "";
+            lastSourceID = "";
+        };
+        const closeSlice = async () => {
+            const activeStage = stage;
+            if (!activeStage)
+                return;
+            await new Promise((resolve, reject) => {
+                activeStage.end((error) => (error ? reject(error) : resolve()));
+            });
+            if (sliceRows) {
+                await writeStagedParquet(stagePath, path);
+                const fileStat = await tryStat(path);
+                const sha256 = await sha256File(path);
+                slices.push({
+                    split,
+                    path,
+                    format: "parquet",
+                    compression: SLICE_COMPRESSION,
+                    rows: sliceRows,
+                    bytes: fileStat?.size ?? 0,
+                    sha256,
+                    first_source_id: firstSourceID,
+                    last_source_id: lastSourceID,
+                });
+            }
+            stage = null;
+            sliceRows = 0;
+        };
+        for await (const row of rows) {
+            if (!path) {
+                await openSlice();
+            }
+            const pq = rowToParquet(row);
+            if (!stage)
+                throw new Error("Parquet slice writer was not opened");
+            await writeStagedRow(stage, pq);
+            sliceRows++;
+            if (sliceRows === 1) {
+                firstSourceID = row.source_id;
+            }
+            lastSourceID = row.source_id;
+            counts[split]++;
+            totalRows++;
+            if (sliceRows >= rowsPerSlice) {
+                await closeSlice();
+                sliceIndex++;
+                path = "";
+            }
+        }
+        await closeSlice();
+    }
+    slices.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)));
+    const manifest = {
+        corpus_version: opts.corpusVersion,
+        schema: PARQUET_COLUMNS,
+        rows_per_slice: rowsPerSlice,
+        row_group_size: ROW_GROUP_SIZE,
+        slices,
+        counts,
+        total_rows: totalRows,
+    };
+    await writeLocalJSONFile(manifest, corpusDir, "MANIFEST.json");
+    return manifest;
+}
+//# sourceMappingURL=parquet.js.map

package/out/utils/parquet.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../lib/utils/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAA;AACnD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AACzG,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,SAAS,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AACrG,OAAO,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AAKpD;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IAClC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAE9C,OAAO,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAmClD;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;oDAEoD;AAEpD;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,wFAAwF;IACxF,+FAA+F;IAC/F,gFAAgF;IAChF,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC5E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AAED,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AACpE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AAEpE,MAAM,oBAAoB,GAAqD;IAC9E,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AAED,SAAS,YAAY,CAAC,IAA2B;IAChD,OAAO,IAAI,UAAU,CAAC;QACrB,GAAG,EAAE,eAAe,CACnB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAC1B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,WAAW,EAAE,eAAe,CAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAClC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,cAAc,CACd;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,IAAI,IAAI,EAAE,CACV;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,IAAI,IAAI,EAAE,CACV;QACD,cAAc,EAAE,eAAe,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,YAAY,EAAE,eAAe,CAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,EAC3C,IAAI,IAAI,EAAE,CACV;QACD,aAAa,EAAE,eAAe,CAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,EAC5C,IAAI,IAAI,EAAE,CACV;KACD,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAA2B,EAAE,IAAY;IAC/E,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,IAAI,uBAAuB,EAAE;SAC9C,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC;SAClC,YAAY,CAAC,WAAW,CAAC;SACzB,kBAAkB,CAAC,cAAc,CAAC;SAClC,KAAK,EAAE,CAAA;IAET,2GAA2G;IAC3G,gHAAgH;IAChH,MAAM,gBAAgB,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,CAAA;AAClE,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC3C,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1F,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAEhE,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,IAAY,EACZ,OAA2B,EAC3B,OAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,UAAU,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAA;IAC5B,MAAM,WAAW,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,aAAa,CAAC,KAAK,CAAC,EAAE,CAAA;IAC/E,MAAM,GAAG,GAAG,UAAU,UAAU,uBAAuB,eAAe,CAAC,IAAI,CAAC,KAAK,WAAW,EAAE,CAAA;IAE9F,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;QAExC,KAAK,IAAI,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,KAAK,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;YAC5G,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,WAAW,CAA8B,CAAA;YAE1E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAM,CAAA;YAC7G,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,OAAO,IAAI,+CAA+C,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACpF,MAAM,OAAO,GACZ,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,oDAAoD,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAErG,MAAM,IAAI,KAAK,CAAC,oEAAoE,OAAO,EAAE,EAAE;gBAC9F,KAAK,EAAE,KAAK;aACZ,CAAC,CAAA;QACH,CAAC;QAED,MAAM,KAAK,CAAA;IACZ,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,SAAS,EAAE,CAAA;IACf,CAAC;AACF,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACzC,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAA;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IACnC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,KAAK,EAAE,CAAC,CAAA;IAE9E,OAAO,KAAK,CAAA;AACb,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,UAAkB;IACtE,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IAEpC,MAAM,cAAc,GACnB,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,OAAO,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IAEjG,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9D,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;YAC5E,aAAa,cAAc,mCAAmC;YAC9D,OAAO,eAAe,CAAC,UAAU,CAAC,yDAAyD,cAAc,GAAG,CAC7G,CAAA;IACF,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,SAAS,EAAE,CAAA;IACf,CAAC;AACF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAAkB,EAAE,GAAe;IAChE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;AACF,CAAC;AA0DD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAA;QAE7E,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,EAAE,CAAA;QAClB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,KAAK,GAAuB,IAAI,CAAA;QACpC,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;YAC/B,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAE5E,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAA;YACjF,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAA;YAC/C,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,MAAM,WAAW,GAAG,KAAK,CAAA;YAEzB,IAAI,CAAC,WAAW;gBAAE,OAAM;YAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,WAAW,CAAC,GAAG,CAAC,CAAC,KAAoB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAC/E,CAAC,CAAC,CAAA;YAEF,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;gBAEzC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;gBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;gBAErC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBAC1B,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YAED,KAAK,GAAG,IAAI,CAAA;YACZ,SAAS,GAAG,CAAC,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,MAAM,SAAS,EAAE,CAAA;YAClB,CAAC;YAED,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YAClE,MAAM,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YAE/B,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACrB,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAC9B,CAAC;YAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAE5B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YAEf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAElB,UAAU,EAAE,CAAA;gBACZ,IAAI,GAAG,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,SAAS,EAAE,eAAe,CAAC,CAAA;IAE9D,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/utils/slice-stats.d.ts

@@ -0,0 +1,52 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ *
+ *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
+ *   and the slice linter (`tools/lint-slice.ts`). The two sides MUST agree on the bigram key format:
+ *   the linter looks its slice's bigrams up in the stats file's `bigrams` table, and a key built with
+ *   a different separator never matches — which silently blanks the bigram-collision check.
+ */
+/**
+ * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`): U+001F UNIT SEPARATOR, a
+ * character no address token contains. Render a key for humans with `key.split(SLICE_STATS_SEP).join(" ")`.
+ */
+export declare const SLICE_STATS_SEP = "\u001F";
+/**
+ * Nested tally: outer key → (inner key → count). Token → label counts, or bigram key → label-bigram counts.
+ */
+type CooccurrenceTable = Map<string, Map<string, number>>;
+/**
+ * The token + bigram co-occurrence tables one pass over labeled rows accumulates.
+ */
+export interface CooccurrenceStats {
+    tokens: CooccurrenceTable;
+    bigrams: CooccurrenceTable;
+}
+/**
+ * An empty pair of tables for {@link accumulateCooccurrences}.
+ */
+export declare function createCooccurrenceStats(): CooccurrenceStats;
+/**
+ * Fold one row's parallel `tokens`/`labels` into the tables. The caller has already checked the arrays are the same
+ * length.
+ */
+export declare function accumulateCooccurrences(stats: CooccurrenceStats, tokens: readonly string[], labels: readonly string[]): void;
+/**
+ * A slice row projected to the two columns the co-occurrence pass reads.
+ */
+export interface TokenLabelRow {
+    tokens: string[];
+    labels: string[];
+    [key: string]: unknown;
+}
+/**
+ * Stream a slice's `tokens`/`labels` columns.
+ *
+ * Projected rather than read whole: parquet is columnar, so the unused columns are never touched. `limit` stops the
+ * iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
+ */
+export declare function streamTokenLabelRows(slicePath: string, limit?: number): AsyncIterable<TokenLabelRow>;
+export {};
+//# sourceMappingURL=slice-stats.d.ts.map

package/out/utils/slice-stats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"slice-stats.d.ts","sourceRoot":"","sources":["../../lib/utils/slice-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;;GAGG;AACH,eAAO,MAAM,eAAe,WAAM,CAAA;AAElC;;GAEG;AACH,KAAK,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;AAEzD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,iBAAiB,CAAA;IACzB,OAAO,EAAE,iBAAiB,CAAA;CAC1B;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,iBAAiB,CAE3D;AAaD;;;GAGG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,MAAM,EAAE,SAAS,MAAM,EAAE,GACvB,IAAI,CAWN;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAuB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,aAAa,CAAC,CAQ3G"}