@mailwoman/corpus 10.0.0 → 10.2.0

package/out/parquet/shuffle.js

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file A bounded streaming shuffle over the rows headed for parquet, so a row-group is a sample of its source.
+ */
+import { shuffleWith } from "@mailwoman/core/random";
+/**
+ * Rows held in memory while shuffling, when the caller names no window.
+ *
+ * Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s
+ * whole 13-country set, while a window below `ROW_GROUP_SIZE` buys no wider mix.
+ */
+export const DEFAULT_SHUFFLE_WINDOW = 250_000;
+/**
+ * Seed for the shuffle when the caller names none, fixed rather than drawn so two builds
+ * of one input write the same row order and a corpus is reproducible from its inputs.
+ */
+export const DEFAULT_SHUFFLE_SEED = 20_260_922;
+/**
+ * Yield the input shuffled within a sliding window of `windowSize` rows, so memory is
+ * bounded by the window rather than the input; `windowSize <= 1` yields the input unchanged
+ * and consumes no random draw, keeping a no-shuffle build byte-identical.
+ */
+export async function* shuffleWithinWindow(rows, random, windowSize = DEFAULT_SHUFFLE_WINDOW) {
+    if (windowSize <= 1) {
+        yield* rows;
+        return;
+    }
+    const buffer = [];
+    for await (const row of rows) {
+        if (buffer.length < windowSize) {
+            buffer.push(row);
+            continue;
+        }
+        const index = Math.floor(random() * buffer.length);
+        const chosen = buffer[index];
+        buffer[index] = row;
+        yield chosen;
+    }
+    shuffleWith(buffer, random);
+    yield* buffer;
+}
+//# sourceMappingURL=shuffle.js.map

package/out/parquet/shuffle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"shuffle.js","sourceRoot":"","sources":["../../lib/parquet/shuffle.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAA;AAEpD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,sBAAsB,GAAG,OAAO,CAAA;AAE7C;;;GAGG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAG,UAAU,CAAA;AAE9C;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,mBAAmB,CACzC,IAAsB,EACtB,MAAoB,EACpB,UAAU,GAAW,sBAAsB;IAE3C,IAAI,UAAU,IAAI,CAAC,EAAE,CAAC;QACrB,KAAK,CAAC,CAAC,IAAI,CAAA;QAEX,OAAM;IACP,CAAC;IAED,MAAM,MAAM,GAAQ,EAAE,CAAA;IAEtB,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QAC9B,IAAI,MAAM,CAAC,MAAM,GAAG,UAAU,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAEhB,SAAQ;QACT,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAA;QAClD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAE,CAAA;QAC7B,MAAM,CAAC,KAAK,CAAC,GAAG,GAAG,CAAA;QAEnB,MAAM,MAAM,CAAA;IACb,CAAC;IAED,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;IAE3B,KAAK,CAAC,CAAC,MAAM,CAAA;AACd,CAAC"}

package/out/parquet/span-sql.d.ts

@@ -0,0 +1,49 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The SQL that reads one component's text out of a corpus row, for a query that runs in DuckDB.
+ *
+ *   A parquet row stores component offsets in `span_starts` and `span_ends` arrays.
+ *   A third array, `span_tags`, identifies each component. The text comes from slicing `raw` at those offsets.
+ *   A query that groups or filters by a
+ *   component performs that slice in SQL instead of decoding every row in the process.
+ *
+ *   `holdoutComponents` in `#tools/overlay/split-slice` performs the same read in JavaScript.
+ *   The SQL and JavaScript operations use different string indexing. The difference is described below.
+ */
+/**
+ * The text of one component tag, or NULL when the row has no span for it.
+ *
+ * `list_position` is one-based and answers NULL for an absent tag.
+ * `substr` takes a one-based start with a length, so the start is `span_starts[i] + 1`
+ * and the length is the span's width.
+ *
+ * Known limitation and scope.
+ * `alignRow` records a span in UTF-16 code units.
+ *
+ * DuckDB's `substr` counts characters.
+ * The two agree for the Basic Multilingual Plane and diverge by one unit per astral character,
+ * so a row carrying one slices late by the number of astral characters before the span.
+ *
+ * In the first 40 train files of `v0.6.0-register-surface`, 6,982 of 40,000,000
+ * rows (0.0175%) contain an astral character.
+ * Among 11,521,648 US rows, 5,230 (0.0454%) contain one.
+ *
+ * `𐍀𐍂𐍉𐍆𐌹𐌳𐌰𐌹𐌽𐍃, RHODE ISLAND` read its region as `ND`, taken from `ISLAND`.
+ *
+ * So this is sound for an aggregate over millions of rows and unsound for a decision about one row.
+ * A caller that acts per row narrows with this and confirms with `holdoutComponents`, which
+ * slices with JavaScript string semantics and therefore reads the offsets as they were written.
+ */
+export declare function componentAtSpanSQL(tag: string, rawColumn?: string): string;
+/**
+ * The same slice against a span index the caller already computed.
+ *
+ * A query can hoist `list_position(span_tags, …)` into a subquery column and read it by name.
+ * A second evaluation in the projection would run the search twice per row.
+ *
+ * Every caveat on {@linkcode componentAtSpanSQL} applies here, because this is the expression it builds.
+ */
+export declare function componentAtIndexSQL(indexExpression: string, rawColumn?: string): string;
+//# sourceMappingURL=span-sql.d.ts.map

package/out/parquet/span-sql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"span-sql.d.ts","sourceRoot":"","sources":["../../lib/parquet/span-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,SAAQ,GAAG,MAAM,CAEzE;AAED;;;;;;;GAOG;AACH,wBAAgB,mBAAmB,CAAC,eAAe,EAAE,MAAM,EAAE,SAAS,SAAQ,GAAG,MAAM,CAEtF"}

package/out/parquet/span-sql.js

@@ -0,0 +1,53 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The SQL that reads one component's text out of a corpus row, for a query that runs in DuckDB.
+ *
+ *   A parquet row stores component offsets in `span_starts` and `span_ends` arrays.
+ *   A third array, `span_tags`, identifies each component. The text comes from slicing `raw` at those offsets.
+ *   A query that groups or filters by a
+ *   component performs that slice in SQL instead of decoding every row in the process.
+ *
+ *   `holdoutComponents` in `#tools/overlay/split-slice` performs the same read in JavaScript.
+ *   The SQL and JavaScript operations use different string indexing. The difference is described below.
+ */
+/**
+ * The text of one component tag, or NULL when the row has no span for it.
+ *
+ * `list_position` is one-based and answers NULL for an absent tag.
+ * `substr` takes a one-based start with a length, so the start is `span_starts[i] + 1`
+ * and the length is the span's width.
+ *
+ * Known limitation and scope.
+ * `alignRow` records a span in UTF-16 code units.
+ *
+ * DuckDB's `substr` counts characters.
+ * The two agree for the Basic Multilingual Plane and diverge by one unit per astral character,
+ * so a row carrying one slices late by the number of astral characters before the span.
+ *
+ * In the first 40 train files of `v0.6.0-register-surface`, 6,982 of 40,000,000
+ * rows (0.0175%) contain an astral character.
+ * Among 11,521,648 US rows, 5,230 (0.0454%) contain one.
+ *
+ * `𐍀𐍂𐍉𐍆𐌹𐌳𐌰𐌹𐌽𐍃, RHODE ISLAND` read its region as `ND`, taken from `ISLAND`.
+ *
+ * So this is sound for an aggregate over millions of rows and unsound for a decision about one row.
+ * A caller that acts per row narrows with this and confirms with `holdoutComponents`, which
+ * slices with JavaScript string semantics and therefore reads the offsets as they were written.
+ */
+export function componentAtSpanSQL(tag, rawColumn = "raw") {
+    return componentAtIndexSQL(`list_position(span_tags, '${tag}')`, rawColumn);
+}
+/**
+ * The same slice against a span index the caller already computed.
+ *
+ * A query can hoist `list_position(span_tags, …)` into a subquery column and read it by name.
+ * A second evaluation in the projection would run the search twice per row.
+ *
+ * Every caveat on {@linkcode componentAtSpanSQL} applies here, because this is the expression it builds.
+ */
+export function componentAtIndexSQL(indexExpression, rawColumn = "raw") {
+    return `CASE WHEN ${indexExpression} IS NULL THEN NULL ELSE substr(${rawColumn}, span_starts[${indexExpression}] + 1, span_ends[${indexExpression}] - span_starts[${indexExpression}]) END`;
+}
+//# sourceMappingURL=span-sql.js.map

package/out/parquet/span-sql.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"span-sql.js","sourceRoot":"","sources":["../../lib/parquet/span-sql.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,MAAM,UAAU,kBAAkB,CAAC,GAAW,EAAE,SAAS,GAAG,KAAK;IAChE,OAAO,mBAAmB,CAAC,6BAA6B,GAAG,IAAI,EAAE,SAAS,CAAC,CAAA;AAC5E,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,mBAAmB,CAAC,eAAuB,EAAE,SAAS,GAAG,KAAK;IAC7E,OAAO,aAAa,eAAe,kCAAkC,SAAS,iBAAiB,eAAe,oBAAoB,eAAe,mBAAmB,eAAe,QAAQ,CAAA;AAC5L,CAAC"}

package/out/parquet/streams.d.ts

@@ -0,0 +1,47 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Pull-based parquet reading, the same split `@mailwoman/core/fs` draws between `/readers` and `/streams`.
+ *
+ *   A stream returns immediately and the work happens as the consumer pulls, so a caller can walk a file larger than
+ *   memory. A caller that wants the rows, or only their count, uses `./readers` instead and awaits a value.
+ */
+import type { PathBuilderLike } from "path-ts";
+/**
+ * DuckDB hands a list column back as `{ items: [...] }`.
+ *
+ * Unwrap it so a row reads the way the schema declares it, recursively,
+ * because a nested list arrives nested the same way.
+ */
+export declare function normalizeDuckDBValue(value: unknown): unknown;
+/**
+ * A row limit has to be a non-negative safe integer before it reaches a `limit` clause,
+ * because it is interpolated rather than bound.
+ */
+export declare function validateRowLimit(value: number): number;
+export interface ParquetRowStreamOptions {
+    /**
+     * Columns to project.
+     *
+     * Every column when omitted.
+     *
+     * A column absent from the file raises rather than coming back absent: a projection that
+     * silently drops a field hands the consumer a well-formed row with the field missing.
+     * The consumer reads that as "this row has no value there" instead of "this file has no such column".
+     */
+    columns?: readonly string[];
+    /**
+     * Stop after this many rows.
+     */
+    limit?: number;
+}
+/**
+ * Open a row stream over a local parquet file, yielding rows in DuckDB-managed chunks.
+ *
+ * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`.
+ * Rows are converted and yielded one chunk at a time, so neither the complete Parquet file
+ * nor the complete result set is copied into JavaScript memory.
+ */
+export declare function openParquetRowStream<T>(path: PathBuilderLike, options?: ParquetRowStreamOptions): AsyncGenerator<T>;
+//# sourceMappingURL=streams.d.ts.map

package/out/parquet/streams.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams.d.ts","sourceRoot":"","sources":["../../lib/parquet/streams.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAI9C;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAoB5D;AAED;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAKtD;AAED,MAAM,WAAW,uBAAuB;IACvC;;;;;;;;OAQG;IACH,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B;;OAEG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACd;AAED;;;;;;GAMG;AACH,wBAAuB,oBAAoB,CAAC,CAAC,EAC5C,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,uBAA4B,GACnC,cAAc,CAAC,CAAC,CAAC,CAgCnB"}

package/out/parquet/streams.js

@@ -0,0 +1,78 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Pull-based parquet reading, the same split `@mailwoman/core/fs` draws between `/readers` and `/streams`.
+ *
+ *   A stream returns immediately and the work happens as the consumer pulls, so a caller can walk a file larger than
+ *   memory. A caller that wants the rows, or only their count, uses `./readers` instead and awaits a value.
+ */
+import { escapeSQLIdentifier, escapeSQLString, openDuckDB } from "#parquet/duckdb";
+/**
+ * DuckDB hands a list column back as `{ items: [...] }`.
+ *
+ * Unwrap it so a row reads the way the schema declares it, recursively,
+ * because a nested list arrives nested the same way.
+ */
+export function normalizeDuckDBValue(value) {
+    if (value && typeof value === "object" && "items" in value && Array.isArray(value.items)) {
+        return value.items.map(normalizeDuckDBValue);
+    }
+    // A DuckDB STRUCT arrives as `{ entries: { field: value } }` rather than as
+    // the plain object its fields describe.
+    // The raw value makes `row.address_levels[0].value` read `undefined` on every row.
+    // A consumer counting values then reports that the file does not populate the column.
+    // Overture's `address_levels` and `sources` are both lists of structs.
+    // Unwrapped values made the audit report all 25,914,431 Italian rows as lacking locality.
+    if (value && typeof value === "object" && "entries" in value && value.entries && typeof value.entries === "object") {
+        return Object.fromEntries(Object.entries(value.entries).map(([key, entry]) => [key, normalizeDuckDBValue(entry)]));
+    }
+    if (Array.isArray(value))
+        return value.map(normalizeDuckDBValue);
+    return value;
+}
+/**
+ * A row limit has to be a non-negative safe integer before it reaches a `limit` clause,
+ * because it is interpolated rather than bound.
+ */
+export function validateRowLimit(value) {
+    if (!Number.isSafeInteger(value) || value < 0)
+        throw new Error(`Parquet row limit must be a non-negative integer: ${value}`);
+    return value;
+}
+/**
+ * Open a row stream over a local parquet file, yielding rows in DuckDB-managed chunks.
+ *
+ * DuckDB opens the path itself and exposes its DataChunks through `fetchChunk()`.
+ * Rows are converted and yielded one chunk at a time, so neither the complete Parquet file
+ * nor the complete result set is copied into JavaScript memory.
+ */
+export async function* openParquetRowStream(path, options = {}) {
+    const { columns, limit } = options;
+    // Disposed when this generator finishes or a consumer abandons it,
+    // since `for await` calls `.return()` on break.
+    using db = await openDuckDB();
+    const projection = columns?.length ? columns.map(escapeSQLIdentifier).join(", ") : "*";
+    const limitClause = limit === undefined ? "" : ` LIMIT ${validateRowLimit(limit)}`;
+    const sql = `SELECT ${projection} FROM read_parquet('${escapeSQLString(path.toString())}')${limitClause}`;
+    try {
+        const stream = await db.stream(sql);
+        const columnNames = stream.columnNames();
+        for (let chunk = await stream.fetchChunk(); chunk && chunk.rowCount > 0; chunk = await stream.fetchChunk()) {
+            const rows = chunk.getRowObjects(columnNames);
+            for (const row of rows) {
+                yield Object.fromEntries(Object.entries(row).map(([key, value]) => [key, normalizeDuckDBValue(value)]));
+            }
+        }
+    }
+    catch (error) {
+        if (columns && /Binder Error|Referenced column|does not exist/.test(String(error))) {
+            const missing = String(error).match(/(?:Referenced column|field named) ["']([^"']+)["']/)?.[1] ?? columns.join(", ");
+            throw new Error(`Parquet projection requested column absent from the file schema: ${missing}`, {
+                cause: error,
+            });
+        }
+        throw error;
+    }
+}
+//# sourceMappingURL=streams.js.map

package/out/parquet/streams.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streams.js","sourceRoot":"","sources":["../../lib/parquet/streams.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAIH,OAAO,EAAE,mBAAmB,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAElF;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAc;IAClD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1F,OAAO,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAC7C,CAAC;IAED,4EAA4E;IAC5E,wCAAwC;IACxC,mFAAmF;IACnF,sFAAsF;IACtF,uEAAuE;IACvE,0FAA0F;IAC1F,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,SAAS,IAAI,KAAK,IAAI,KAAK,CAAC,OAAO,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACpH,OAAO,MAAM,CAAC,WAAW,CACxB,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,OAAkC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAClH,CAAA;IACF,CAAC;IAED,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IAEhE,OAAO,KAAK,CAAA;AACb,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC7C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,qDAAqD,KAAK,EAAE,CAAC,CAAA;IAE9E,OAAO,KAAK,CAAA;AACb,CAAC;AAmBD;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,oBAAoB,CAC1C,IAAqB,EACrB,OAAO,GAA4B,EAAE;IAErC,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAA;IAClC,mEAAmE;IACnE,gDAAgD;IAChD,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAC7B,MAAM,UAAU,GAAG,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAA;IACtF,MAAM,WAAW,GAAG,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAA;IAClF,MAAM,GAAG,GAAG,UAAU,UAAU,uBAAuB,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,KAAK,WAAW,EAAE,CAAA;IAEzG,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACnC,MAAM,WAAW,GAAG,MAAM,CAAC,WAAW,EAAE,CAAA;QAExC,KAAK,IAAI,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,KAAK,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,MAAM,CAAC,UAAU,EAAE,EAAE,CAAC;YAC5G,MAAM,IAAI,GAAG,KAAK,CAAC,aAAa,CAAC,WAAW,CAA8B,CAAA;YAE1E,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACxB,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,EAAE,oBAAoB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAM,CAAA;YAC7G,CAAC;QACF,CAAC;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,IAAI,OAAO,IAAI,+CAA+C,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;YACpF,MAAM,OAAO,GACZ,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,oDAAoD,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAErG,MAAM,IAAI,KAAK,CAAC,oEAAoE,OAAO,EAAE,EAAE;gBAC9F,KAAK,EAAE,KAAK;aACZ,CAAC,CAAA;QACH,CAAC;QAED,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC"}

package/out/parquet/writers.d.ts

@@ -0,0 +1,124 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Writes Parquet files and the manifest that describes their contents.
+ *
+ *   This module exposes two writers. {@linkcode writeParquetFile} writes one file from rows already in memory.
+ *   {@linkcode writeParquetSplits} streams per-split row iterables into as many files as the row cap requires. It
+ *   returns the manifest. Both writers create their parent directory, as `@mailwoman/core/fs` writers do
+ *   encode rather than repeat at each call site.
+ *
+ *   Layout under `<outputDir>`:
+ *
+ *   ```
+ *   corpus-v<version>/
+ *     manifest.json
+ *     train/
+ *       part-0000.parquet
+ *       part-0001.parquet
+ *     val/
+ *       part-0000.parquet
+ *     test/
+ *       part-0000.parquet
+ *   ```
+ *
+ *   Each file caps at `rowsPerFile` (default 1,000,000). Within a file DuckDB writes a row group every
+ *   {@linkcode ROW_GROUP_SIZE} rows. The manifest captures every file's path, row count, byte size and SHA-256, the
+ *   digest computed by re-reading the file once after close — cheap against the cost of writing it.
+ *
+ *   A part is closed by its row count and never by a source boundary, so one part may hold the tail of one source and
+ *   the head of the next. A reader must therefore take a file's sources from all of its rows.
+ */
+import { type PathBuilderLike } from "path-ts";
+import { PARQUET_COMPRESSION, type ParquetRow } from "#parquet/schema";
+import type { LabeledRow } from "#types";
+import type { SplitName } from "#utils/split";
+/**
+ * Write one parquet file from rows held in memory.
+ */
+export declare function writeParquetFile(rows: readonly ParquetRow[], path: PathBuilderLike): Promise<void>;
+/**
+ * Per-file metadata captured in `manifest.json`, one entry per `.parquet` file of a split.
+ *
+ * The `slices` key it sits under is the wire interface the Python loader reads
+ * (`manifest_files` in `corpus_files.py`, with its pre-rename fallback); every corpus
+ * on disk includes it, so the key name is not the writer's to change.
+ */
+export interface ParquetFileDescriptor {
+    split: SplitName;
+    path: string;
+    format: "parquet";
+    compression: typeof PARQUET_COMPRESSION;
+    rows: number;
+    bytes: number;
+    sha256: string;
+    first_source_id: string;
+    last_source_id: string;
+    /**
+     * The file's corpus source slug, when the writer knows it.
+     *
+     * `audit.ts` prefers this over inferring the source from `first_source_id`'s prefix;
+     * {@linkcode writeParquetSplits} itself writes multi-source files and leaves it unset.
+     */
+    source?: string;
+}
+export interface ParquetManifest {
+    corpus_version: string;
+    schema: readonly string[];
+    rows_per_slice: number;
+    row_group_size: number;
+    slices: ParquetFileDescriptor[];
+    counts: Record<SplitName, number>;
+    total_rows: number;
+}
+export interface WriteParquetSplitsOptions {
+    /**
+     * Root output directory.
+     *
+     * The corpus version directory is created beneath it.
+     */
+    outputDir: PathBuilderLike;
+    /**
+     * Corpus version stamped onto rows and into the output directory name.
+     */
+    corpusVersion: string;
+    /**
+     * Max rows per `.parquet` file.
+     *
+     * @defaultValue `1,000,000`
+     */
+    rowsPerFile?: number;
+    /**
+     * Continue from the files `<corpusDir>/MANIFEST.json` already records.
+     *
+     * The manifest is rewritten after each finished file.
+     * A descriptor is appended only after its `.parquet` file closes, so each recorded file is complete.
+     *
+     * A resumed run rereads each descriptor's size.
+     * It discards that many rows from the front of each split's iterable and opens the next file index.
+     *
+     * The caller therefore has to supply the same iterables in the same order.
+     * For `buildCorpus` that means the same labeled files and the same shuffle seed.
+     */
+    resume?: boolean;
+}
+/**
+ * Pre-partitioned labeled-row streams, one per split.
+ *
+ * Callers (`buildCorpus`) decide each row's split inline at align time via `splitForRow`
+ * and route rows to the matching stream.
+ * Splits with no rows can be omitted or passed as an empty iterable.
+ * {@linkcode writeParquetSplits} skips them.
+ */
+export type PerSplitRows = Partial<Record<SplitName, AsyncIterable<LabeledRow>>>;
+/**
+ * Streams labeled rows into `.parquet` files, with one set per split.
+ * It writes the manifest that describes them.
+ *
+ * Splits are processed sequentially so only one file is open at a time.
+ * Rows are staged to newline-delimited JSON with backpressure, then DuckDB
+ * writes the Parquet file from disk.
+ */
+export declare function writeParquetSplits(perSplit: PerSplitRows, opts: WriteParquetSplitsOptions): Promise<ParquetManifest>;
+//# sourceMappingURL=writers.d.ts.map

package/out/parquet/writers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"writers.d.ts","sourceRoot":"","sources":["../../lib/parquet/writers.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAYH,OAAO,EAAE,KAAK,eAAe,EAAsB,MAAM,SAAS,CAAA;AAIlE,OAAO,EAGN,mBAAmB,EACnB,KAAK,UAAU,EAIf,MAAM,iBAAiB,CAAA;AACxB,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AA0E7C;;GAEG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,SAAS,UAAU,EAAE,EAAE,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAexG;AAED;;;;;;GAMG;AACH,MAAM,WAAW,qBAAqB;IACrC,KAAK,EAAE,SAAS,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,SAAS,CAAA;IACjB,WAAW,EAAE,OAAO,mBAAmB,CAAA;IACvC,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,eAAe,EAAE,MAAM,CAAA;IACvB,cAAc,EAAE,MAAM,CAAA;IACtB;;;;;OAKG;IACH,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;AAED,MAAM,WAAW,eAAe;IAC/B,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB,MAAM,EAAE,qBAAqB,EAAE,CAAA;IAC/B,MAAM,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAA;IACjC,UAAU,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,yBAAyB;IACzC;;;;OAIG;IACH,SAAS,EAAE,eAAe,CAAA;IAE1B;;OAEG;IACH,aAAa,EAAE,MAAM,CAAA;IAErB;;;;OAIG;IACH,WAAW,CAAC,EAAE,MAAM,CAAA;IAEpB;;;;;;;;;;;OAWG;IACH,MAAM,CAAC,EAAE,OAAO,CAAA;CAChB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,YAAY,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,CAAA;AA0BhF;;;;;;;GAOG;AACH,wBAAsB,kBAAkB,CACvC,QAAQ,EAAE,YAAY,EACtB,IAAI,EAAE,yBAAyB,GAC7B,OAAO,CAAC,eAAe,CAAC,CAqJ1B"}