@mailwoman/corpus 9.4.0 → 10.1.0

package/out/utils/parquet.js

@@ -1,371 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Final output sliceer for the corpus pipeline.
- *
- *   Phase 1 (#9) shipped JSONL slices + a Python (PyArrow) converter as the path to binary Parquet —
- *   bridging until the JS toolchain caught up. Phase 1.5 (#18 §4) replaced that with a native JS
- *   writer. The build pipeline no longer touches Python at all in its hot path; the only remaining
- *   Python is the one-shot `train_tokenizer.py` SentencePiece step.
- *
- *   Compression: `SNAPPY`. The plan in #18 §4 specified `zstd`; parquet-wasm supports SNAPPY,
- *   which is the standard ML-corpus default (and PyArrow's default). Documented in
- *   `DECISIONS.md`.
- *
- *   Layout under `<outputDir>`:
- *
- *   ```
- *   corpus-v<version>/
- *   MANIFEST.json
- *   train/
- *     part-0000.parquet
- *     part-0001.parquet
- *     ...
- *   val/
- *     part-0000.parquet
- *   test/
- *     part-0000.parquet
- * ```
- *
- *   Each slice caps at `rowsPerSlice` (default 1_000_000); within a slice, DuckDB writes row
- *   groups every `ROW_GROUP_SIZE` (50_000) rows per the issue spec. The MANIFEST captures every
- *   slice's path, row count, byte size, and SHA-256 (computed by re-reading the slice once after
- *   close — cheap relative to writing it).
- */
-import { tryStat } from "@mailwoman/core/fs/readers";
-import { openWriteStream } from "@mailwoman/core/fs/streams";
-import { temporaryDirectory } from "@mailwoman/core/fs/temporary";
-import { writeLocalBuffer, writeLocalJSONFile, makeDirectories } from "@mailwoman/core/fs/writers";
-import { sha256File } from "@mailwoman/core/hash";
-import { once } from "@mailwoman/core/utils/events";
-import { Field, Int32, List, Table as ArrowTable, tableToIPC, Utf8, vectorFromArray } from "apache-arrow";
-import { Compression, Table as WasmTable, WriterPropertiesBuilder, writeParquet } from "parquet-wasm";
-import { join } from "path-ts";
-/**
- * Row groups are written at this cadence within a slice.
- */
-export const ROW_GROUP_SIZE = 50_000;
-/**
- * Escape `value` for a single-quoted SQL string literal; the caller supplies the quotes.
- */
-export function escapeSQLString(value) {
-    return value.replaceAll("'", "''");
-}
-/**
- * Open an in-memory DuckDB connection. `@duckdb/node-api` is an optional peer — lazy import (the pipeline convention),
- * so the heavy native module loads only on the paths that read or write Parquet through DuckDB.
- */
-export async function connectDuckDB() {
-    const { DuckDBInstance } = await import("@duckdb/node-api");
-    const instance = await DuckDBInstance.create();
-    return await instance.connect();
-}
-/**
- * Snappy is the codec selected for corpus slices.
- */
-export const SLICE_COMPRESSION = "SNAPPY";
-/**
- * Column names emitted into every slice. Matches `ParquetRow`.
- */
-export const PARQUET_COLUMNS = [
-    "raw",
-    "tokens",
-    "labels",
-    "span_starts",
-    "span_ends",
-    "span_tags",
-    "country",
-    "locale",
-    "source",
-    "source_id",
-    "corpus_version",
-    "license",
-    "synth_method",
-    "synth_base_id",
-];
-/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member,
-   not a text-encoding identifier. Lowercasing it does not type-check against ParquetSchemaDefinition,
-   and the rule has no way to tell the two apart. */
-/**
- * Parquet schema for `LabeledRow` per #18 §4. Optional fields use `optional: true`; repeated UTF8 columns capture
- * tokens/labels arrays. Compression is per-column SNAPPY.
- */
-export const LABELED_ROW_SCHEMA = {
-    raw: { type: "UTF8", compression: SLICE_COMPRESSION },
-    tokens: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    labels: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    // v0.5.0 char-offset label spans (#519): parallel arrays over `raw` (UTF-16 code units,
-    // [start, end) exclusive-end, sorted, non-overlapping). INT32 — raw is a short address string,
-    // and INT32 round-trips as `number` where parquetjs INT64 would surface bigint.
-    span_starts: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
-    span_ends: { type: "INT32", repeated: true, compression: SLICE_COMPRESSION },
-    span_tags: { type: "UTF8", repeated: true, compression: SLICE_COMPRESSION },
-    country: { type: "UTF8", compression: SLICE_COMPRESSION },
-    locale: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-    source: { type: "UTF8", compression: SLICE_COMPRESSION },
-    source_id: { type: "UTF8", compression: SLICE_COMPRESSION },
-    corpus_version: { type: "UTF8", compression: SLICE_COMPRESSION },
-    license: { type: "UTF8", compression: SLICE_COMPRESSION },
-    synth_method: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-    synth_base_id: { type: "UTF8", compression: SLICE_COMPRESSION, optional: true },
-};
-const stringListType = new List(new Field("item", new Utf8(), true));
-const int32ListType = new List(new Field("item", new Int32(), true));
-const PARQUET_COLUMN_TYPES = {
-    raw: "VARCHAR",
-    tokens: "VARCHAR[]",
-    labels: "VARCHAR[]",
-    span_starts: "INTEGER[]",
-    span_ends: "INTEGER[]",
-    span_tags: "VARCHAR[]",
-    country: "VARCHAR",
-    locale: "VARCHAR",
-    source: "VARCHAR",
-    source_id: "VARCHAR",
-    corpus_version: "VARCHAR",
-    license: "VARCHAR",
-    synth_method: "VARCHAR",
-    synth_base_id: "VARCHAR",
-};
-function parquetTable(rows) {
-    return new ArrowTable({
-        raw: vectorFromArray(rows.map((row) => row.raw), new Utf8()),
-        tokens: vectorFromArray(rows.map((row) => row.tokens), stringListType),
-        labels: vectorFromArray(rows.map((row) => row.labels), stringListType),
-        span_starts: vectorFromArray(rows.map((row) => row.span_starts), int32ListType),
-        span_ends: vectorFromArray(rows.map((row) => row.span_ends), int32ListType),
-        span_tags: vectorFromArray(rows.map((row) => row.span_tags), stringListType),
-        country: vectorFromArray(rows.map((row) => row.country), new Utf8()),
-        locale: vectorFromArray(rows.map((row) => row.locale ?? null), new Utf8()),
-        source: vectorFromArray(rows.map((row) => row.source), new Utf8()),
-        source_id: vectorFromArray(rows.map((row) => row.source_id), new Utf8()),
-        corpus_version: vectorFromArray(rows.map((row) => row.corpus_version), new Utf8()),
-        license: vectorFromArray(rows.map((row) => row.license), new Utf8()),
-        synth_method: vectorFromArray(rows.map((row) => row.synth_method ?? null), new Utf8()),
-        synth_base_id: vectorFromArray(rows.map((row) => row.synth_base_id ?? null), new Utf8()),
-    });
-}
-export async function writeParquetRows(rows, path) {
-    const arrow = parquetTable(rows);
-    const wasmTable = WasmTable.fromIPCStream(tableToIPC(arrow, "stream"));
-    const properties = new WriterPropertiesBuilder()
-        .setCompression(Compression.SNAPPY)
-        .setCreatedBy("mailwoman")
-        .setMaxRowGroupSize(ROW_GROUP_SIZE)
-        .build();
-    // parquet-wasm serializes key-value metadata through a hash map, whose order is not stable between writes.
-    // Slice identity and provenance live in MANIFEST.json, so omitting file metadata preserves deterministic bytes.
-    await writeLocalBuffer(writeParquet(wasmTable, properties), path);
-}
-function normalizeDuckDBValue(value) {
-    if (value && typeof value === "object" && "items" in value && Array.isArray(value.items)) {
-        return value.items.map(normalizeDuckDBValue);
-    }
-    if (Array.isArray(value))
-        return value.map(normalizeDuckDBValue);
-    return value;
-}
-/**
- * Stream rows from a local Parquet file in DuckDB-managed chunks.
- *
- * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`. Rows are converted and yielded one
- * chunk at a time; the complete Parquet file and complete result set are never copied into JavaScript memory.
- */
-export async function* streamParquetRows(path, columns, options) {
-    const db = await connectDuckDB();
-    const projection = columns?.length ? columns.map(escapeSQLIdentifier).join(", ") : "*";
-    const limit = options?.limit;
-    const limitClause = limit === undefined ? "" : ` LIMIT ${validateLimit(limit)}`;
-    const sql = `SELECT ${projection} FROM read_parquet('${escapeSQLString(path)}')${limitClause}`;
-    try {
-        const stream = await db.stream(sql);
-        const columnNames = stream.columnNames();
-        for (let chunk = await stream.fetchChunk(); chunk && chunk.rowCount > 0; chunk = await stream.fetchChunk()) {
-            const rows = chunk.getRowObjects(columnNames);
-            for (const row of rows) {
-                yield Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeDuckDBValue(value)]));
-            }
-        }
-    }
-    catch (error) {
-        if (columns && /Binder Error|Referenced column|does not exist/.test(String(error))) {
-            const missing = String(error).match(/(?:Referenced column|field named) ["']([^"']+)["']/)?.[1] ?? columns.join(", ");
-            throw new Error(`Parquet projection requested column absent from the file schema: ${missing}`, {
-                cause: error,
-            });
-        }
-        throw error;
-    }
-    finally {
-        db.closeSync();
-    }
-}
-function escapeSQLIdentifier(value) {
-    return `"${value.replaceAll('"', '""')}"`;
-}
-function validateLimit(value) {
-    if (!Number.isSafeInteger(value) || value < 0)
-        throw new Error(`Parquet row limit must be a non-negative integer: ${value}`);
-    return value;
-}
-async function writeStagedParquet(stagePath, outputPath) {
-    const db = await connectDuckDB();
-    const columns = [...PARQUET_COLUMNS];
-    const columnsLiteral = "{" + columns.map((column) => `'${column}': '${PARQUET_COLUMN_TYPES[column]}'`).join(", ") + "}";
-    const selectList = columns.map(escapeSQLIdentifier).join(", ");
-    try {
-        await db.run("SET preserve_insertion_order=true");
-        await db.run(`COPY (SELECT ${selectList} FROM read_json('${escapeSQLString(stagePath)}', ` +
-            `columns = ${columnsLiteral}, format = 'newline_delimited')) ` +
-            `TO '${escapeSQLString(outputPath)}' (FORMAT PARQUET, COMPRESSION SNAPPY, ROW_GROUP_SIZE ${ROW_GROUP_SIZE})`);
-    }
-    finally {
-        db.closeSync();
-    }
-}
-async function writeStagedRow(stage, row) {
-    if (!stage.write(JSON.stringify(row) + "\n")) {
-        await once(stage, "drain");
-    }
-}
-/**
- * Project a labeled row to the Parquet schema.
- *
- * The span triple is REQUIRED here (#519): `alignRow` emits it on every labeled row, so a row arriving without it came
- * from a producer that hasn't migrated — writing it would silently drop the v0.5.0 labels from the slice (the "builders
- * before parquet = silent loss" hazard). Loud failure, naming the row, instead.
- */
-export function rowToParquet(row) {
-    const { span_starts, span_ends, span_tags } = row;
-    if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
-        throw new Error(`rowToParquet: row is missing the char-offset span triple (#519) — ` +
-            `span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
-            `(source=${row.source}, source_id=${row.source_id}). ` +
-            `Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
-            `producers that emit tokens/labels only have not migrated to the v0.5.0 format.`);
-    }
-    if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
-        throw new Error(`rowToParquet: span triple arrays are not parallel — ` +
-            `starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
-            `(source=${row.source}, source_id=${row.source_id})`);
-    }
-    return {
-        raw: row.raw,
-        tokens: row.tokens,
-        labels: row.labels,
-        span_starts,
-        span_ends,
-        span_tags,
-        country: row.country,
-        locale: row.locale ?? null,
-        source: row.source,
-        source_id: row.source_id,
-        corpus_version: row.corpus_version,
-        license: row.license,
-        synth_method: row.synth?.method ?? null,
-        synth_base_id: row.synth?.base_source_id ?? null,
-    };
-}
-/**
- * Stream labeled rows into `.parquet` slices, one set of slices per split. Splits are processed sequentially so that
- * only one slice is open at a time. Rows are staged to newline-delimited JSON with backpressure, then DuckDB writes the
- * Parquet file from disk.
- *
- * Callers pass per-split `AsyncIterable<LabeledRow>` (`PerSplitRows`); the prior `splitFor(sourceID)` callback is gone
- * because pre-partitioning at the caller eliminates the O(n) `Map<source_id, SplitName>` it required. See `buildCorpus`
- * for the new wire-up.
- */
-export async function writeSlices(perSplit, opts) {
-    const rowsPerSlice = opts.rowsPerSlice ?? 1_000_000;
-    const corpusDir = join(opts.outputDir, `corpus-v${opts.corpusVersion}`);
-    await makeDirectories(corpusDir);
-    const slices = [];
-    const counts = { train: 0, val: 0, test: 0 };
-    let totalRows = 0;
-    for (const split of ["train", "val", "test"]) {
-        const rows = perSplit[split];
-        if (!rows)
-            continue;
-        await using staging = await temporaryDirectory(`mailwoman-parquet-${split}-`);
-        let sliceIndex = 0;
-        let path = "";
-        let stagePath = "";
-        let sliceRows = 0;
-        let stage = null;
-        let firstSourceID = "";
-        let lastSourceID = "";
-        const openSlice = async () => {
-            const splitDir = join(corpusDir, split);
-            await makeDirectories(splitDir);
-            path = join(splitDir, `part-${String(sliceIndex).padStart(4, "0")}.parquet`);
-            stagePath = staging.resolve(`part-${String(sliceIndex).padStart(4, "0")}.ndjson`);
-            stage = staging.use(openWriteStream(stagePath));
-            sliceRows = 0;
-            firstSourceID = "";
-            lastSourceID = "";
-        };
-        const closeSlice = async () => {
-            const activeStage = stage;
-            if (!activeStage)
-                return;
-            await new Promise((resolve, reject) => {
-                activeStage.end((error) => (error ? reject(error) : resolve()));
-            });
-            if (sliceRows) {
-                await writeStagedParquet(stagePath, path);
-                const fileStat = await tryStat(path);
-                const sha256 = await sha256File(path);
-                slices.push({
-                    split,
-                    path,
-                    format: "parquet",
-                    compression: SLICE_COMPRESSION,
-                    rows: sliceRows,
-                    bytes: fileStat?.size ?? 0,
-                    sha256,
-                    first_source_id: firstSourceID,
-                    last_source_id: lastSourceID,
-                });
-            }
-            stage = null;
-            sliceRows = 0;
-        };
-        for await (const row of rows) {
-            if (!path) {
-                await openSlice();
-            }
-            const pq = rowToParquet(row);
-            if (!stage)
-                throw new Error("Parquet slice writer was not opened");
-            await writeStagedRow(stage, pq);
-            sliceRows++;
-            if (sliceRows === 1) {
-                firstSourceID = row.source_id;
-            }
-            lastSourceID = row.source_id;
-            counts[split]++;
-            totalRows++;
-            if (sliceRows >= rowsPerSlice) {
-                await closeSlice();
-                sliceIndex++;
-                path = "";
-            }
-        }
-        await closeSlice();
-    }
-    slices.sort((a, b) => (a.split === b.split ? a.path.localeCompare(b.path) : a.split.localeCompare(b.split)));
-    const manifest = {
-        corpus_version: opts.corpusVersion,
-        schema: PARQUET_COLUMNS,
-        rows_per_slice: rowsPerSlice,
-        row_group_size: ROW_GROUP_SIZE,
-        slices,
-        counts,
-        total_rows: totalRows,
-    };
-    await writeLocalJSONFile(manifest, corpusDir, "MANIFEST.json");
-    return manifest;
-}
-//# sourceMappingURL=parquet.js.map

package/out/utils/parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parquet.js","sourceRoot":"","sources":["../../lib/utils/parquet.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmCG;AAEH,OAAO,EAAE,OAAO,EAAE,MAAM,4BAA4B,CAAA;AACpD,OAAO,EAAE,eAAe,EAAoB,MAAM,4BAA4B,CAAA;AAC9E,OAAO,EAAE,kBAAkB,EAAE,MAAM,8BAA8B,CAAA;AACjE,OAAO,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAA;AAClG,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAA;AACjD,OAAO,EAAE,IAAI,EAAE,MAAM,8BAA8B,CAAA;AACnD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,IAAI,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,cAAc,CAAA;AACzG,OAAO,EAAE,WAAW,EAAE,KAAK,IAAI,SAAS,EAAE,uBAAuB,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AACrG,OAAO,EAAE,IAAI,EAAwB,MAAM,SAAS,CAAA;AAKpD;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAQD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IAClC,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,CAAA;IAE9C,OAAO,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAG,QAAiB,CAAA;AAmClD;;GAEG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,cAAc;IACd,eAAe;CACN,CAAA;AAEV;;oDAEoD;AAEpD;;;GAGG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACrD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxE,wFAAwF;IACxF,+FAA+F;IAC/F,gFAAgF;IAChF,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC5E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IACxE,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACxD,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAC3D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IAChE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE;IACzD,YAAY,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC9E,aAAa,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAC/E,CAAA;AAED,MAAM,cAAc,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AACpE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,KAAK,EAAE,EAAE,IAAI,CAAC,CAAC,CAAA;AAEpE,MAAM,oBAAoB,GAAqD;IAC9E,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACxB,CAAA;AAED,SAAS,YAAY,CAAC,IAA2B;IAChD,OAAO,IAAI,UAAU,CAAC;QACrB,GAAG,EAAE,eAAe,CACnB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,EAC1B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,cAAc,CACd;QACD,WAAW,EAAE,eAAe,CAC3B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,WAAW,CAAC,EAClC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,aAAa,CACb;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,cAAc,CACd;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,IAAI,IAAI,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,MAAM,EAAE,eAAe,CACtB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,EAC7B,IAAI,IAAI,EAAE,CACV;QACD,SAAS,EAAE,eAAe,CACzB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,CAAC,EAChC,IAAI,IAAI,EAAE,CACV;QACD,cAAc,EAAE,eAAe,CAC9B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,CAAC,EACrC,IAAI,IAAI,EAAE,CACV;QACD,OAAO,EAAE,eAAe,CACvB,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAC9B,IAAI,IAAI,EAAE,CACV;QACD,YAAY,EAAE,eAAe,CAC5B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,YAAY,IAAI,IAAI,CAAC,EAC3C,IAAI,IAAI,EAAE,CACV;QACD,aAAa,EAAE,eAAe,CAC7B,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,aAAa,IAAI,IAAI,CAAC,EAC5C,IAAI,IAAI,EAAE,CACV;KACD,CAAC,CAAA;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAA2B,EAAE,IAAY;IAC/E,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,CAAC,UAAU,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,CAAA;IAEtE,MAAM,UAAU,GAAG,IAAI,uBAAuB,EAAE;SAC9C,cAAc,CAAC,WAAW,CAAC,MAAM,CAAC;SAClC,YAAY,CAAC,WAAW,CAAC;SACzB,kBAAkB,CAAC,cAAc,CAAC;SAClC,KAAK,EAAE,CAAA;IAET,2GAA2G;IAC3G,gHAAgH;IAChH,MAAM,gBAAgB,CAAC,YAAY,CAAC,SAAS,EAAE,UAAU,CAAC,EAAE,IAAI,CAAC,CAAA;AAClE,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc;IAC3C,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1F,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAC7C,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAEhE,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iBAAiB,CACvC,IAAY,EACZ,OAA2B,EAC3B,OAA4B;IAE5B,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,UAAU,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtF,MAAM,KAAK,GAAG,OAAO,EAAE,KAAK,CAAA;IAC5B,MAAM,WAAW,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,aAAa,CAAC,KAAK,CAAC,EAAE,CAAA;IAC/E,MAAM,GAAG,GAAG,UAAU,UAAU,uBAAuB,eAAe,CAAC,IAAI,CAAC,KAAK,WAAW,EAAE,CAAA;IAE9F,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;QAExC,KAAK,IAAI,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,KAAK,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;YAC5G,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,WAAW,CAA8B,CAAA;YAE1E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAM,CAAA;YAC7G,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,OAAO,IAAI,+CAA+C,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACpF,MAAM,OAAO,GACZ,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,oDAAoD,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAErG,MAAM,IAAI,KAAK,CAAC,oEAAoE,OAAO,EAAE,EAAE;gBAC9F,KAAK,EAAE,KAAK;aACZ,CAAC,CAAA;QACH,CAAC;QAED,MAAM,KAAK,CAAA;IACZ,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,SAAS,EAAE,CAAA;IACf,CAAC;AACF,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACzC,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAA;AAC1C,CAAC;AAED,SAAS,aAAa,CAAC,KAAa;IACnC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,KAAK,EAAE,CAAC,CAAA;IAE9E,OAAO,KAAK,CAAA;AACb,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,SAAiB,EAAE,UAAkB;IACtE,MAAM,EAAE,GAAG,MAAM,aAAa,EAAE,CAAA;IAChC,MAAM,OAAO,GAAG,CAAC,GAAG,eAAe,CAAC,CAAA;IAEpC,MAAM,cAAc,GACnB,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,MAAM,OAAO,oBAAoB,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAA;IAEjG,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IAE9D,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAA;QAEjD,MAAM,EAAE,CAAC,GAAG,CACX,gBAAgB,UAAU,oBAAoB,eAAe,CAAC,SAAS,CAAC,KAAK;YAC5E,aAAa,cAAc,mCAAmC;YAC9D,OAAO,eAAe,CAAC,UAAU,CAAC,yDAAyD,cAAc,GAAG,CAC7G,CAAA;IACF,CAAC;YAAS,CAAC;QACV,EAAE,CAAC,SAAS,EAAE,CAAA;IACf,CAAC;AACF,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,KAAkB,EAAE,GAAe;IAChE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC;QAC9C,MAAM,IAAI,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC3B,CAAC;AACF,CAAC;AA0DD;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,GAAG,CAAC,KAAK,EAAE,MAAM,IAAI,IAAI;QACvC,aAAa,EAAE,GAAG,CAAC,KAAK,EAAE,cAAc,IAAI,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,QAAsB,EAAE,IAAwB;IACjF,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,IAAI,SAAS,CAAA;IACnD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,IAAI,CAAC,aAAa,EAAE,CAAC,CAAA;IACvE,MAAM,eAAe,CAAC,SAAS,CAAC,CAAA;IAEhC,MAAM,MAAM,GAAsB,EAAE,CAAA;IACpC,MAAM,MAAM,GAA8B,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,CAAA;IACvE,IAAI,SAAS,GAAG,CAAC,CAAA;IAEjB,KAAK,MAAM,KAAK,IAAI,CAAC,OAAO,EAAE,KAAK,EAAE,MAAM,CAAU,EAAE,CAAC;QACvD,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAA;QAE5B,IAAI,CAAC,IAAI;YAAE,SAAQ;QAEnB,YAAY,OAAO,GAAG,MAAM,kBAAkB,CAAC,qBAAqB,KAAK,GAAG,CAAC,CAAA;QAE7E,IAAI,UAAU,GAAG,CAAC,CAAA;QAClB,IAAI,IAAI,GAAG,EAAE,CAAA;QACb,IAAI,SAAS,GAAG,EAAE,CAAA;QAClB,IAAI,SAAS,GAAG,CAAC,CAAA;QACjB,IAAI,KAAK,GAAuB,IAAI,CAAA;QACpC,IAAI,aAAa,GAAG,EAAE,CAAA;QACtB,IAAI,YAAY,GAAG,EAAE,CAAA;QAErB,MAAM,SAAS,GAAG,KAAK,IAAmB,EAAE;YAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;YACvC,MAAM,eAAe,CAAC,QAAQ,CAAC,CAAA;YAC/B,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAA;YAE5E,SAAS,GAAG,OAAO,CAAC,OAAO,CAAC,QAAQ,MAAM,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAC,CAAA;YACjF,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC,CAAA;YAC/C,SAAS,GAAG,CAAC,CAAA;YACb,aAAa,GAAG,EAAE,CAAA;YAClB,YAAY,GAAG,EAAE,CAAA;QAClB,CAAC,CAAA;QAED,MAAM,UAAU,GAAG,KAAK,IAAmB,EAAE;YAC5C,MAAM,WAAW,GAAG,KAAK,CAAA;YAEzB,IAAI,CAAC,WAAW;gBAAE,OAAM;YAExB,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC3C,WAAW,CAAC,GAAG,CAAC,CAAC,KAAoB,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;YAC/E,CAAC,CAAC,CAAA;YAEF,IAAI,SAAS,EAAE,CAAC;gBACf,MAAM,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAA;gBAEzC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAA;gBACpC,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,CAAA;gBAErC,MAAM,CAAC,IAAI,CAAC;oBACX,KAAK;oBACL,IAAI;oBACJ,MAAM,EAAE,SAAS;oBACjB,WAAW,EAAE,iBAAiB;oBAC9B,IAAI,EAAE,SAAS;oBACf,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;oBAC1B,MAAM;oBACN,eAAe,EAAE,aAAa;oBAC9B,cAAc,EAAE,YAAY;iBAC5B,CAAC,CAAA;YACH,CAAC;YAED,KAAK,GAAG,IAAI,CAAA;YACZ,SAAS,GAAG,CAAC,CAAA;QACd,CAAC,CAAA;QAED,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,MAAM,SAAS,EAAE,CAAA;YAClB,CAAC;YAED,MAAM,EAAE,GAAG,YAAY,CAAC,GAAG,CAAC,CAAA;YAE5B,IAAI,CAAC,KAAK;gBAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAA;YAClE,MAAM,cAAc,CAAC,KAAK,EAAE,EAAE,CAAC,CAAA;YAE/B,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;gBACrB,aAAa,GAAG,GAAG,CAAC,SAAS,CAAA;YAC9B,CAAC;YAED,YAAY,GAAG,GAAG,CAAC,SAAS,CAAA;YAE5B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAA;YAEf,SAAS,EAAE,CAAA;YAEX,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;gBAC/B,MAAM,UAAU,EAAE,CAAA;gBAElB,UAAU,EAAE,CAAA;gBACZ,IAAI,GAAG,EAAE,CAAA;YACV,CAAC;QACF,CAAC;QAED,MAAM,UAAU,EAAE,CAAA;IACnB,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;IAE5G,MAAM,QAAQ,GAAkB;QAC/B,cAAc,EAAE,IAAI,CAAC,aAAa;QAClC,MAAM,EAAE,eAAe;QACvB,cAAc,EAAE,YAAY;QAC5B,cAAc,EAAE,cAAc;QAC9B,MAAM;QACN,MAAM;QACN,UAAU,EAAE,SAAS;KACrB,CAAA;IAED,MAAM,kBAAkB,CAAC,QAAQ,EAAE,SAAS,EAAE,eAAe,CAAC,CAAA;IAE9D,OAAO,QAAQ,CAAA;AAChB,CAAC"}

package/out/utils/slice-stats.d.ts

@@ -1,52 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
- *   and the slice linter (`tools/lint-slice.ts`). The two sides MUST agree on the bigram key format:
- *   the linter looks its slice's bigrams up in the stats file's `bigrams` table, and a key built with
- *   a different separator never matches — which silently blanks the bigram-collision check.
- */
-/**
- * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`): U+001F UNIT SEPARATOR, a
- * character no address token contains. Render a key for humans with `key.split(SLICE_STATS_SEP).join(" ")`.
- */
-export declare const SLICE_STATS_SEP = "\u001F";
-/**
- * Nested tally: outer key → (inner key → count). Token → label counts, or bigram key → label-bigram counts.
- */
-type CooccurrenceTable = Map<string, Map<string, number>>;
-/**
- * The token + bigram co-occurrence tables one pass over labeled rows accumulates.
- */
-export interface CooccurrenceStats {
-    tokens: CooccurrenceTable;
-    bigrams: CooccurrenceTable;
-}
-/**
- * An empty pair of tables for {@link accumulateCooccurrences}.
- */
-export declare function createCooccurrenceStats(): CooccurrenceStats;
-/**
- * Fold one row's parallel `tokens`/`labels` into the tables. The caller has already checked the arrays are the same
- * length.
- */
-export declare function accumulateCooccurrences(stats: CooccurrenceStats, tokens: readonly string[], labels: readonly string[]): void;
-/**
- * A slice row projected to the two columns the co-occurrence pass reads.
- */
-export interface TokenLabelRow {
-    tokens: string[];
-    labels: string[];
-    [key: string]: unknown;
-}
-/**
- * Stream a slice's `tokens`/`labels` columns.
- *
- * Projected rather than read whole: parquet is columnar, so the unused columns are never touched. `limit` stops the
- * iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
- */
-export declare function streamTokenLabelRows(slicePath: string, limit?: number): AsyncIterable<TokenLabelRow>;
-export {};
-//# sourceMappingURL=slice-stats.d.ts.map

package/out/utils/slice-stats.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"slice-stats.d.ts","sourceRoot":"","sources":["../../lib/utils/slice-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH;;;GAGG;AACH,eAAO,MAAM,eAAe,WAAM,CAAA;AAElC;;GAEG;AACH,KAAK,iBAAiB,GAAG,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAA;AAEzD;;GAEG;AACH,MAAM,WAAW,iBAAiB;IACjC,MAAM,EAAE,iBAAiB,CAAA;IACzB,OAAO,EAAE,iBAAiB,CAAA;CAC1B;AAED;;GAEG;AACH,wBAAgB,uBAAuB,IAAI,iBAAiB,CAE3D;AAaD;;;GAGG;AACH,wBAAgB,uBAAuB,CACtC,KAAK,EAAE,iBAAiB,EACxB,MAAM,EAAE,SAAS,MAAM,EAAE,EACzB,MAAM,EAAE,SAAS,MAAM,EAAE,GACvB,IAAI,CAWN;AAED;;GAEG;AACH,MAAM,WAAW,aAAa;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;;;;GAKG;AACH,wBAAuB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,aAAa,CAAC,aAAa,CAAC,CAQ3G"}

package/out/utils/slice-stats.js

@@ -1,58 +0,0 @@
-/**
- * @copyright Sister Software
- * @license AGPL-3.0
- * @author Teffen Ellis, et al.
- *
- *   Token/label co-occurrence accounting shared by the corpus-stats builder (`tools/corpus-stats.ts`)
- *   and the slice linter (`tools/lint-slice.ts`). The two sides MUST agree on the bigram key format:
- *   the linter looks its slice's bigrams up in the stats file's `bigrams` table, and a key built with
- *   a different separator never matches — which silently blanks the bigram-collision check.
- */
-import { streamParquetRows } from "#utils/parquet";
-/**
- * Separator inside a bigram key (`tok1␟tok2`) and a label-bigram value (`lab1␟lab2`): U+001F UNIT SEPARATOR, a
- * character no address token contains. Render a key for humans with `key.split(SLICE_STATS_SEP).join(" ")`.
- */
-export const SLICE_STATS_SEP = "";
-/**
- * An empty pair of tables for {@link accumulateCooccurrences}.
- */
-export function createCooccurrenceStats() {
-    return { tokens: new Map(), bigrams: new Map() };
-}
-function bump(table, key, sub) {
-    let counter = table.get(key);
-    if (!counter) {
-        counter = new Map();
-        table.set(key, counter);
-    }
-    counter.set(sub, (counter.get(sub) ?? 0) + 1);
-}
-/**
- * Fold one row's parallel `tokens`/`labels` into the tables. The caller has already checked the arrays are the same
- * length.
- */
-export function accumulateCooccurrences(stats, tokens, labels) {
-    for (let i = 0; i < tokens.length; i++) {
-        const tk = tokens[i];
-        const lb = labels[i];
-        bump(stats.tokens, tk, lb);
-        if (i + 1 < tokens.length) {
-            bump(stats.bigrams, tk + SLICE_STATS_SEP + tokens[i + 1], lb + SLICE_STATS_SEP + labels[i + 1]);
-        }
-    }
-}
-/**
- * Stream a slice's `tokens`/`labels` columns.
- *
- * Projected rather than read whole: parquet is columnar, so the unused columns are never touched. `limit` stops the
- * iteration rather than filtering afterwards, so a capped run reads only the row groups it needs.
- */
-export async function* streamTokenLabelRows(slicePath, limit) {
-    let emitted = 0;
-    for await (const row of streamParquetRows(slicePath, ["tokens", "labels"], { limit })) {
-        yield row;
-        emitted++;
-    }
-}
-//# sourceMappingURL=slice-stats.js.map

package/out/utils/slice-stats.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"slice-stats.js","sourceRoot":"","sources":["../../lib/utils/slice-stats.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAA;AAElD;;;GAGG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG,GAAG,CAAA;AAelC;;GAEG;AACH,MAAM,UAAU,uBAAuB;IACtC,OAAO,EAAE,MAAM,EAAE,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,IAAI,GAAG,EAAE,EAAE,CAAA;AACjD,CAAC;AAED,SAAS,IAAI,CAAC,KAAwB,EAAE,GAAW,EAAE,GAAW;IAC/D,IAAI,OAAO,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAE5B,IAAI,CAAC,OAAO,EAAE,CAAC;QACd,OAAO,GAAG,IAAI,GAAG,EAAE,CAAA;QACnB,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;IACxB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;AAC9C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CACtC,KAAwB,EACxB,MAAyB,EACzB,MAAyB;IAEzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QACrB,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAA;QAErB,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC,CAAA;QAE1B,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,EAAE,GAAG,eAAe,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,EAAE,EAAE,GAAG,eAAe,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,CAAA;QAClG,CAAC;IACF,CAAC;AACF,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CAAC,SAAiB,EAAE,KAAc;IAC5E,IAAI,OAAO,GAAG,CAAC,CAAA;IAEf,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,iBAAiB,CAAgB,SAAS,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;QACtG,MAAM,GAAG,CAAA;QAET,OAAO,EAAE,CAAA;IACV,CAAC;AACF,CAAC"}