@mailwoman/corpus 9.4.0 → 10.1.0

package/out/parquet/duckdb.d.ts

@@ -0,0 +1,100 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The DuckDB boundary for the parquet family — the one place that opens a connection and the two escapers every
+ *   statement built here goes through.
+ *
+ *   `@duckdb/node-api` is an optional peer, so the import is lazy: the native module loads only on the paths that read
+ *   or write Parquet through DuckDB. A consumer that only needs the schema types never loads it.
+ */
+/**
+ * An open DuckDB connection, re-exported so consumers of {@link openDuckDB} can name
+ * the type without their own static dependency on the optional peer.
+ */
+export type { DuckDBConnection } from "@duckdb/node-api";
+/**
+ * An in-memory DuckDB connection whose disposal also closes the instance behind it.
+ *
+ * Both closes are synchronous, so this is `Disposable` rather than `AsyncDisposable`,
+ * the same as {@link DatabaseClient}.
+ */
+export type DisposableDuckDB = import("@duckdb/node-api").DuckDBConnection & Disposable;
+/**
+ * The share of host memory one DuckDB instance may hold.
+ *
+ * DuckDB allocates outside the V8 heap, so `--max-old-space-size` does not bound it and a query
+ * over a corpus with hundreds of millions of rows will exhaust the host rather than spill.
+ * A `COUNT(DISTINCT …)` over `v0.6.0-register-surface`'s 697,675,170 train rows did
+ * that twice on 2026-09-28, with a second corpus tool running beside it.
+ *
+ * The default is a quarter because it has to bound the host rather than one query.
+ * Two instances at half the host exceed it together.
+ *
+ * That case took the machine down.
+ * A quarter admits four concurrent instances and still leaves room for the page cache the
+ * parquet reads run through and for the Node heap of each process holding a connection.
+ *
+ * Past its limit DuckDB spills to `temp_directory`.
+ * The query finishes slower.
+ * That is the failure to prefer.
+ *
+ * A caller that knows it holds the host by itself passes a larger `memoryLimitBytes`.
+ */
+export declare const DUCKDB_MEMORY_SHARE = 0.25;
+/**
+ * Per-instance settings, applied on every connection this module opens.
+ */
+export interface DuckDBLimits {
+    /**
+     * Bytes one instance may hold before it spills.
+     * Defaults to {@linkcode DUCKDB_MEMORY_SHARE} of host memory.
+     */
+    memoryLimitBytes?: number;
+    /**
+     * Worker threads.
+     * Defaults to DuckDB's own choice.
+     *
+     * A build phase that runs DuckDB beside other work sets this, because each thread
+     * holds its own share of the memory limit and the default takes every core.
+     */
+    threads?: number;
+    /**
+     * Where a spill is written.
+     * Defaults to the data root's `tmp/duckdb`.
+     *
+     * The default sits under the data root rather than in the system temporary directory,
+     * because a spill from a corpus-sized query is tens of gigabytes and `/tmp` is commonly a
+     * memory-backed filesystem, where spilling to it consumes the memory the spill exists to release.
+     */
+    temporaryDirectory?: string;
+}
+/**
+ * Open an in-memory DuckDB connection, taken with `using db = await openDuckDB()`.
+ *
+ * The connection is returned directly, so it reads as a connection at every call site.
+ * Disposal closes the connection and then the instance.
+ *
+ * An open connection holds the native instance for the life of the process.
+ * A connection opened per file in a loop holds one instance per file.
+ *
+ * Every connection has a memory limit.
+ * It is set here rather than at each call site because host memory exhaustion affects
+ * every process, including processes unrelated to the query that caused it.
+ */
+export declare function openDuckDB(limits?: DuckDBLimits): Promise<DisposableDuckDB>;
+/**
+ * Escape `value` for a single-quoted SQL string literal.
+ *
+ * The caller supplies the quotes.
+ */
+export declare function escapeSQLString(value: string): string;
+/**
+ * Escape `value` as a double-quoted SQL identifier, for a column name that reaches a statement from data.
+ *
+ * A projection names columns the caller chose, so the name is not a literal this module wrote.
+ * Quotes preserve a column whose name collides with a keyword — or includes a space —
+ * from re-parsing as syntax.
+ */
+export declare function escapeSQLIdentifier(value: string): string;
+//# sourceMappingURL=duckdb.d.ts.map

package/out/parquet/duckdb.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckdb.d.ts","sourceRoot":"","sources":["../../lib/parquet/duckdb.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH;;;GAGG;AACH,YAAY,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAA;AAExD;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG,OAAO,kBAAkB,EAAE,gBAAgB,GAAG,UAAU,CAAA;AAEvF;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,eAAO,MAAM,mBAAmB,OAAO,CAAA;AAEvC;;GAEG;AACH,MAAM,WAAW,YAAY;IAC5B;;;OAGG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAA;IAEzB;;;;;;OAMG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAEhB;;;;;;;OAOG;IACH,kBAAkB,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAsB,UAAU,CAAC,MAAM,GAAE,YAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAgCrF;AAED;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAErD;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAEzD"}

package/out/parquet/duckdb.js

@@ -0,0 +1,92 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The DuckDB boundary for the parquet family — the one place that opens a connection and the two escapers every
+ *   statement built here goes through.
+ *
+ *   `@duckdb/node-api` is an optional peer, so the import is lazy: the native module loads only on the paths that read
+ *   or write Parquet through DuckDB. A consumer that only needs the schema types never loads it.
+ */
+/**
+ * The share of host memory one DuckDB instance may hold.
+ *
+ * DuckDB allocates outside the V8 heap, so `--max-old-space-size` does not bound it and a query
+ * over a corpus with hundreds of millions of rows will exhaust the host rather than spill.
+ * A `COUNT(DISTINCT …)` over `v0.6.0-register-surface`'s 697,675,170 train rows did
+ * that twice on 2026-09-28, with a second corpus tool running beside it.
+ *
+ * The default is a quarter because it has to bound the host rather than one query.
+ * Two instances at half the host exceed it together.
+ *
+ * That case took the machine down.
+ * A quarter admits four concurrent instances and still leaves room for the page cache the
+ * parquet reads run through and for the Node heap of each process holding a connection.
+ *
+ * Past its limit DuckDB spills to `temp_directory`.
+ * The query finishes slower.
+ * That is the failure to prefer.
+ *
+ * A caller that knows it holds the host by itself passes a larger `memoryLimitBytes`.
+ */
+export const DUCKDB_MEMORY_SHARE = 0.25;
+/**
+ * Open an in-memory DuckDB connection, taken with `using db = await openDuckDB()`.
+ *
+ * The connection is returned directly, so it reads as a connection at every call site.
+ * Disposal closes the connection and then the instance.
+ *
+ * An open connection holds the native instance for the life of the process.
+ * A connection opened per file in a loop holds one instance per file.
+ *
+ * Every connection has a memory limit.
+ * It is set here rather than at each call site because host memory exhaustion affects
+ * every process, including processes unrelated to the query that caused it.
+ */
+export async function openDuckDB(limits = {}) {
+    const { DuckDBInstance } = await import("@duckdb/node-api");
+    const { dataRootPath } = await import("@mailwoman/core/data-root");
+    const { ByteFormatter } = await import("@mailwoman/core/fs/formatters");
+    const { makeDirectories } = await import("@mailwoman/core/fs/writers");
+    const { totalMemoryBytes } = await import("@mailwoman/core/utils/system");
+    const memoryLimitBytes = limits.memoryLimitBytes ?? Math.floor(totalMemoryBytes() * DUCKDB_MEMORY_SHARE);
+    const temporaryDirectory = limits.temporaryDirectory ?? dataRootPath("tmp", "duckdb").toString();
+    // DuckDB writes a spill file rather than creating the directory, so a missing
+    // one turns a spill into a failed query.
+    // Created here, where the default is chosen.
+    await makeDirectories(temporaryDirectory);
+    const instance = await DuckDBInstance.create(":memory:", {
+        // DuckDB parses the value, so it takes a string with a unit rather than a byte count.
+        // `formatIEC` writes the binary unit DuckDB reads back verbatim; `formatSI`'s `MB`
+        // would be read as decimal megabytes and set a limit 4.9% under the one this asked for.
+        memory_limit: ByteFormatter.formatIEC(memoryLimitBytes),
+        temp_directory: temporaryDirectory,
+        ...(limits.threads ? { threads: String(limits.threads) } : {}),
+    });
+    const connection = await instance.connect();
+    return Object.assign(connection, {
+        [Symbol.dispose]: () => {
+            connection.closeSync();
+            instance.closeSync();
+        },
+    });
+}
+/**
+ * Escape `value` for a single-quoted SQL string literal.
+ *
+ * The caller supplies the quotes.
+ */
+export function escapeSQLString(value) {
+    return value.replaceAll("'", "''");
+}
+/**
+ * Escape `value` as a double-quoted SQL identifier, for a column name that reaches a statement from data.
+ *
+ * A projection names columns the caller chose, so the name is not a literal this module wrote.
+ * Quotes preserve a column whose name collides with a keyword — or includes a space —
+ * from re-parsing as syntax.
+ */
+export function escapeSQLIdentifier(value) {
+    return `"${value.replaceAll('"', '""')}"`;
+}
+//# sourceMappingURL=duckdb.js.map

package/out/parquet/duckdb.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"duckdb.js","sourceRoot":"","sources":["../../lib/parquet/duckdb.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAgBH;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAA;AAgCvC;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,MAAM,GAAiB,EAAE;IACzD,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;IAC3D,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAA;IAClE,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAA;IACvE,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAA;IACtE,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAA;IAEzE,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,mBAAmB,CAAC,CAAA;IACxG,MAAM,kBAAkB,GAAG,MAAM,CAAC,kBAAkB,IAAI,YAAY,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC,QAAQ,EAAE,CAAA;IAEhG,8EAA8E;IAC9E,yCAAyC;IACzC,6CAA6C;IAC7C,MAAM,eAAe,CAAC,kBAAkB,CAAC,CAAA;IAEzC,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,CAAC,UAAU,EAAE;QACxD,sFAAsF;QACtF,mFAAmF;QACnF,wFAAwF;QACxF,YAAY,EAAE,aAAa,CAAC,SAAS,CAAC,gBAAgB,CAAC;QACvD,cAAc,EAAE,kBAAkB;QAClC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC9D,CAAC,CAAA;IAEF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;IAE3C,OAAO,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE;QAChC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,GAAG,EAAE;YACtB,UAAU,CAAC,SAAS,EAAE,CAAA;YACtB,QAAQ,CAAC,SAAS,EAAE,CAAA;QACrB,CAAC;KACD,CAAC,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,eAAe,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAA;AACnC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,KAAa;IAChD,OAAO,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAA;AAC1C,CAAC"}

package/out/parquet/readers.d.ts

@@ -0,0 +1,44 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Promise-based parquet reads. The iterator-based reads live in `./streams`.
+ *
+ *   {@linkcode readParquetRows} throws when the file is missing. {@linkcode tryReadParquetRows} returns `null`.
+ *   Both throw for an unreadable file, so a caller cannot mistake a read failure for an empty file.
+ */
+import type { PathBuilderLike } from "path-ts";
+import { type ParquetRowStreamOptions } from "#parquet/streams";
+/**
+ * Reads every row of a parquet file into memory.
+ *
+ * Use {@linkcode openParquetRowStream} for a file too large to hold in memory.
+ *
+ * @throws When the file is missing or `columns` includes a column the file lacks.
+ */
+export declare function readParquetRows<T>(path: PathBuilderLike, options?: ParquetRowStreamOptions): Promise<T[]>;
+/**
+ * Reads every row of a parquet file, or returns `null` when the file is missing.
+ *
+ * @throws When the file cannot be parsed or `columns` includes a column the file lacks.
+ */
+export declare function tryReadParquetRows<T>(path: PathBuilderLike, options?: ParquetRowStreamOptions): Promise<T[] | null>;
+/**
+ * Counts the rows of a parquet file from its metadata, so the cost stays flat as the row count grows.
+ *
+ * The function opens its own DuckDB connection.
+ *
+ * @throws When the file is missing.
+ */
+export declare function countParquetRows(path: PathBuilderLike): Promise<number>;
+/**
+ * Returns a parquet file's column names in file order.
+ *
+ * The query uses `DESCRIBE`, which lists logical columns.
+ * `parquet_schema` lists physical leaves instead, where a list column such as
+ * `tokens` appears only as its `element` child.
+ *
+ * @throws When the file is missing.
+ */
+export declare function parquetColumnNames(path: PathBuilderLike): Promise<string[]>;
+//# sourceMappingURL=readers.d.ts.map

package/out/parquet/readers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"readers.d.ts","sourceRoot":"","sources":["../../lib/parquet/readers.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,SAAS,CAAA;AAG9C,OAAO,EAAwB,KAAK,uBAAuB,EAAE,MAAM,kBAAkB,CAAA;AAErF;;;;;;GAMG;AACH,wBAAsB,eAAe,CAAC,CAAC,EAAE,IAAI,EAAE,eAAe,EAAE,OAAO,GAAE,uBAA4B,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAMnH;AAED;;;;GAIG;AACH,wBAAsB,kBAAkB,CAAC,CAAC,EACzC,IAAI,EAAE,eAAe,EACrB,OAAO,GAAE,uBAA4B,GACnC,OAAO,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAIrB;AAED;;;;;;GAMG;AACH,wBAAsB,gBAAgB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAY7E;AAED;;;;;;;;GAQG;AACH,wBAAsB,kBAAkB,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAUjF"}

package/out/parquet/readers.js

@@ -0,0 +1,69 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file Promise-based parquet reads. The iterator-based reads live in `./streams`.
+ *
+ *   {@linkcode readParquetRows} throws when the file is missing. {@linkcode tryReadParquetRows} returns `null`.
+ *   Both throw for an unreadable file, so a caller cannot mistake a read failure for an empty file.
+ */
+import { pathExists } from "@mailwoman/core/fs/readers";
+import { escapeSQLString, openDuckDB } from "#parquet/duckdb";
+import { openParquetRowStream } from "#parquet/streams";
+/**
+ * Reads every row of a parquet file into memory.
+ *
+ * Use {@linkcode openParquetRowStream} for a file too large to hold in memory.
+ *
+ * @throws When the file is missing or `columns` includes a column the file lacks.
+ */
+export async function readParquetRows(path, options = {}) {
+    if (!(await pathExists(path))) {
+        throw new Error(`No parquet file at ${path}`);
+    }
+    return await Array.fromAsync(openParquetRowStream(path, options));
+}
+/**
+ * Reads every row of a parquet file, or returns `null` when the file is missing.
+ *
+ * @throws When the file cannot be parsed or `columns` includes a column the file lacks.
+ */
+export async function tryReadParquetRows(path, options = {}) {
+    if (!(await pathExists(path)))
+        return null;
+    return await Array.fromAsync(openParquetRowStream(path, options));
+}
+/**
+ * Counts the rows of a parquet file from its metadata, so the cost stays flat as the row count grows.
+ *
+ * The function opens its own DuckDB connection.
+ *
+ * @throws When the file is missing.
+ */
+export async function countParquetRows(path) {
+    if (!(await pathExists(path))) {
+        throw new Error(`No parquet file at ${path}`);
+    }
+    using db = await openDuckDB();
+    const result = await db.runAndReadAll(`SELECT count(*) AS n FROM read_parquet('${escapeSQLString(path.toString())}')`);
+    const rows = result.getRowObjects();
+    return Number(rows[0]?.n ?? 0);
+}
+/**
+ * Returns a parquet file's column names in file order.
+ *
+ * The query uses `DESCRIBE`, which lists logical columns.
+ * `parquet_schema` lists physical leaves instead, where a list column such as
+ * `tokens` appears only as its `element` child.
+ *
+ * @throws When the file is missing.
+ */
+export async function parquetColumnNames(path) {
+    if (!(await pathExists(path))) {
+        throw new Error(`No parquet file at ${path}`);
+    }
+    using db = await openDuckDB();
+    const result = await db.runAndReadAll(`DESCRIBE SELECT * FROM read_parquet('${escapeSQLString(path.toString())}')`);
+    return result.getRowObjects().map((row) => String(row.column_name));
+}
+//# sourceMappingURL=readers.js.map

package/out/parquet/readers.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"readers.js","sourceRoot":"","sources":["../../lib/parquet/readers.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,4BAA4B,CAAA;AAGvD,OAAO,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAA;AAC7D,OAAO,EAAE,oBAAoB,EAAgC,MAAM,kBAAkB,CAAA;AAErF;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAI,IAAqB,EAAE,OAAO,GAA4B,EAAE;IACpG,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAA;IAC9C,CAAC;IAED,OAAO,MAAM,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAI,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;AACrE,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,IAAqB,EACrB,OAAO,GAA4B,EAAE;IAErC,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAA;IAE1C,OAAO,MAAM,KAAK,CAAC,SAAS,CAAC,oBAAoB,CAAI,IAAI,EAAE,OAAO,CAAC,CAAC,CAAA;AACrE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,IAAqB;IAC3D,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,2CAA2C,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAA;IAEtH,MAAM,IAAI,GAAG,MAAM,CAAC,aAAa,EAA2B,CAAA;IAE5D,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAA;AAC/B,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAqB;IAC7D,IAAI,CAAC,CAAC,MAAM,UAAU,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,sBAAsB,IAAI,EAAE,CAAC,CAAA;IAC9C,CAAC;IAED,MAAM,EAAE,GAAG,MAAM,UAAU,EAAE,CAAA;IAE7B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,aAAa,CAAC,wCAAwC,eAAe,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAA;IAEnH,OAAQ,MAAM,CAAC,aAAa,EAAsC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAA;AACzG,CAAC"}

package/out/parquet/schema.d.ts

@@ -0,0 +1,95 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The corpus parquet schema defines its columns and logical types. It also defines the projection from a `LabeledRow`.
+ *
+ *   Every file in this family agrees on this one definition. A column added here and nowhere else fails to compile
+ *   against {@linkcode ParquetRow}, which keeps the writer, the reader and the manifest's `schema` key from drifting
+ *   apart.
+ *
+ *   Compression is `snappy` throughout. PyArrow uses it by default. It is also the standard ML-corpus codec. A reader
+ *   outside this repository therefore opens the files without configuration.
+ */
+import type { LabeledRow } from "#types";
+/**
+ * Row groups are written at this cadence within a parquet file.
+ */
+export declare const ROW_GROUP_SIZE = 50000;
+/**
+ * Rows per parquet file when a writer's caller names no limit.
+ *
+ * A file is closed here rather than grown, because `writeParquetFile` materializes
+ * one Arrow table and Arrow's list builder raises before a whole source fits:
+ * 4,228,212 rows raised where 1,603,143 wrote.
+ * The manifest records the value in force as `rows_per_slice`.
+ */
+export declare const ROWS_PER_FILE = 1000000;
+/**
+ * Snappy is the codec selected for corpus parquet files.
+ */
+export declare const PARQUET_COMPRESSION = "SNAPPY";
+export interface ParquetFieldDefinition {
+    type: "UTF8" | "INT32";
+    compression: typeof PARQUET_COMPRESSION;
+    repeated?: boolean;
+    optional?: boolean;
+}
+export type ParquetSchemaDefinition<T> = Record<Extract<keyof T, string>, ParquetFieldDefinition>;
+/**
+ * A single Parquet row shape.
+ *
+ * The index signature allows callers to retain source fields before projection.
+ *
+ * Optional fields are represented as null in the Arrow table and read back as null.
+ */
+export interface ParquetRow {
+    raw: string;
+    tokens: readonly string[];
+    labels: readonly string[];
+    span_starts: readonly number[];
+    span_ends: readonly number[];
+    span_tags: readonly string[];
+    country: string;
+    locale?: string | null;
+    source: string;
+    source_id: string;
+    corpus_version: string;
+    license: string;
+    register?: string | null;
+    surface: string;
+    recipe?: string | null;
+    base_source_id?: string | null;
+    [key: string]: unknown;
+}
+/**
+ * Column names emitted into every parquet file.
+ *
+ * Matches `ParquetRow`.
+ */
+export declare const PARQUET_COLUMNS: readonly ["raw", "tokens", "labels", "span_starts", "span_ends", "span_tags", "country", "locale", "source", "source_id", "corpus_version", "license", "register", "surface", "recipe", "base_source_id"];
+/**
+ * The DuckDB type each column is read and written as.
+ *
+ * Paired with {@linkcode PARQUET_COLUMNS} so a `read_json` column map and a `copy`
+ * select list are built from one list rather than two that can disagree.
+ */
+export declare const PARQUET_COLUMN_TYPES: Record<(typeof PARQUET_COLUMNS)[number], string>;
+/**
+ * Parquet schema for `LabeledRow`.
+ *
+ * Optional fields set `optional: true`.
+ * Repeated UTF8 columns capture the tokens and labels arrays.
+ */
+export declare const LABELED_ROW_SCHEMA: ParquetSchemaDefinition<ParquetRow>;
+/**
+ * Project a labeled row to the Parquet schema.
+ *
+ * The span triple is required because `alignRow` emits it on every labeled row.
+ * A row without it came from a producer that has not migrated.
+ *
+ * The writer would drop the labels from the file if it wrote that row.
+ * A thrown error identifies the row instead.
+ */
+export declare function rowToParquet(row: LabeledRow): ParquetRow;
+//# sourceMappingURL=schema.d.ts.map

package/out/parquet/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../lib/parquet/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAA;AAExC;;GAEG;AACH,eAAO,MAAM,cAAc,QAAS,CAAA;AAEpC;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa,UAAY,CAAA;AAEtC;;GAEG;AACH,eAAO,MAAM,mBAAmB,WAAW,CAAA;AAE3C,MAAM,WAAW,sBAAsB;IAEtC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAA;IACtB,WAAW,EAAE,OAAO,mBAAmB,CAAA;IACvC,QAAQ,CAAC,EAAE,OAAO,CAAA;IAClB,QAAQ,CAAC,EAAE,OAAO,CAAA;CAClB;AAED,MAAM,MAAM,uBAAuB,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,sBAAsB,CAAC,CAAA;AAEjG;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IAC1B,GAAG,EAAE,MAAM,CAAA;IACX,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,MAAM,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,WAAW,EAAE,SAAS,MAAM,EAAE,CAAA;IAC9B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IAC5B,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,OAAO,EAAE,MAAM,CAAA;IACf,QAAQ,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACxB,OAAO,EAAE,MAAM,CAAA;IACf,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,GAAG,IAAI,CAAA;IAC9B,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;CACtB;AAED;;;;GAIG;AACH,eAAO,MAAM,eAAe,YAC3B,KAAK,EACL,QAAQ,EACR,QAAQ,EACR,aAAa,EACb,WAAW,EACX,WAAW,EACX,SAAS,EACT,QAAQ,EACR,QAAQ,EACR,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,UAAU,EACV,SAAS,EACT,QAAQ,EACR,gBAAgB,CACP,CAAA;AAEV;;;;;GAKG;AACH,eAAO,MAAM,oBAAoB,EAAE,MAAM,CAAC,CAAC,OAAO,eAAe,CAAC,CAAC,MAAM,CAAC,EAAE,MAAM,CAiBjF,CAAA;AAKD;;;;;GAKG;AACH,eAAO,MAAM,kBAAkB,EAAE,uBAAuB,CAAC,UAAU,CAkBlE,CAAA;AAID;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAmDxD"}

package/out/parquet/schema.js

@@ -0,0 +1,157 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file The corpus parquet schema defines its columns and logical types. It also defines the projection from a `LabeledRow`.
+ *
+ *   Every file in this family agrees on this one definition. A column added here and nowhere else fails to compile
+ *   against {@linkcode ParquetRow}, which keeps the writer, the reader and the manifest's `schema` key from drifting
+ *   apart.
+ *
+ *   Compression is `snappy` throughout. PyArrow uses it by default. It is also the standard ML-corpus codec. A reader
+ *   outside this repository therefore opens the files without configuration.
+ */
+/**
+ * Row groups are written at this cadence within a parquet file.
+ */
+export const ROW_GROUP_SIZE = 50_000;
+/**
+ * Rows per parquet file when a writer's caller names no limit.
+ *
+ * A file is closed here rather than grown, because `writeParquetFile` materializes
+ * one Arrow table and Arrow's list builder raises before a whole source fits:
+ * 4,228,212 rows raised where 1,603,143 wrote.
+ * The manifest records the value in force as `rows_per_slice`.
+ */
+export const ROWS_PER_FILE = 1_000_000;
+/**
+ * Snappy is the codec selected for corpus parquet files.
+ */
+export const PARQUET_COMPRESSION = "SNAPPY";
+/**
+ * Column names emitted into every parquet file.
+ *
+ * Matches `ParquetRow`.
+ */
+export const PARQUET_COLUMNS = [
+    "raw",
+    "tokens",
+    "labels",
+    "span_starts",
+    "span_ends",
+    "span_tags",
+    "country",
+    "locale",
+    "source",
+    "source_id",
+    "corpus_version",
+    "license",
+    "register",
+    "surface",
+    "recipe",
+    "base_source_id",
+];
+/**
+ * The DuckDB type each column is read and written as.
+ *
+ * Paired with {@linkcode PARQUET_COLUMNS} so a `read_json` column map and a `copy`
+ * select list are built from one list rather than two that can disagree.
+ */
+export const PARQUET_COLUMN_TYPES = {
+    raw: "VARCHAR",
+    tokens: "VARCHAR[]",
+    labels: "VARCHAR[]",
+    span_starts: "INTEGER[]",
+    span_ends: "INTEGER[]",
+    span_tags: "VARCHAR[]",
+    country: "VARCHAR",
+    locale: "VARCHAR",
+    source: "VARCHAR",
+    source_id: "VARCHAR",
+    corpus_version: "VARCHAR",
+    license: "VARCHAR",
+    register: "VARCHAR",
+    surface: "VARCHAR",
+    recipe: "VARCHAR",
+    base_source_id: "VARCHAR",
+};
+/* oxlint-disable unicorn/text-encoding-identifier-case -- `"UTF8"` below is a ParquetType enum member rather than a text-encoding identifier. A lowercase spelling does not type-check against ParquetSchemaDefinition.
+   The rule cannot distinguish the enum member from a text-encoding identifier. */
+/**
+ * Parquet schema for `LabeledRow`.
+ *
+ * Optional fields set `optional: true`.
+ * Repeated UTF8 columns capture the tokens and labels arrays.
+ */
+export const LABELED_ROW_SCHEMA = {
+    raw: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    tokens: { type: "UTF8", repeated: true, compression: PARQUET_COMPRESSION },
+    labels: { type: "UTF8", repeated: true, compression: PARQUET_COMPRESSION },
+    // Char-offset label spans, meaning parallel arrays over `raw` in UTF-16 code units. Each span is [start, end) with an exclusive end, sorted and non-overlapping. INT32 holds a short address string and round-trips as `number` where parquetjs INT64 would surface bigint.
+    span_starts: { type: "INT32", repeated: true, compression: PARQUET_COMPRESSION },
+    span_ends: { type: "INT32", repeated: true, compression: PARQUET_COMPRESSION },
+    span_tags: { type: "UTF8", repeated: true, compression: PARQUET_COMPRESSION },
+    country: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    locale: { type: "UTF8", compression: PARQUET_COMPRESSION, optional: true },
+    source: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    source_id: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    corpus_version: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    license: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    register: { type: "UTF8", compression: PARQUET_COMPRESSION, optional: true },
+    surface: { type: "UTF8", compression: PARQUET_COMPRESSION },
+    recipe: { type: "UTF8", compression: PARQUET_COMPRESSION, optional: true },
+    base_source_id: { type: "UTF8", compression: PARQUET_COMPRESSION, optional: true },
+};
+/* oxlint-enable unicorn/text-encoding-identifier-case */
+/**
+ * Project a labeled row to the Parquet schema.
+ *
+ * The span triple is required because `alignRow` emits it on every labeled row.
+ * A row without it came from a producer that has not migrated.
+ *
+ * The writer would drop the labels from the file if it wrote that row.
+ * A thrown error identifies the row instead.
+ */
+export function rowToParquet(row) {
+    const { span_starts, span_ends, span_tags } = row;
+    if (span_starts === undefined || span_ends === undefined || span_tags === undefined) {
+        throw new Error(`rowToParquet: row is missing the char-offset span triple (#519) — ` +
+            `span_starts=${span_starts !== undefined} span_ends=${span_ends !== undefined} span_tags=${span_tags !== undefined} ` +
+            `(source=${row.source}, source_id=${row.source_id}). ` +
+            `Every parquet-bound row must carry span_starts/span_ends/span_tags; ` +
+            `producers that emit tokens/labels only have not migrated to the v0.5.0 format.`);
+    }
+    if (span_starts.length !== span_ends.length || span_starts.length !== span_tags.length) {
+        throw new Error(`rowToParquet: span triple arrays are not parallel — ` +
+            `starts=${span_starts.length} ends=${span_ends.length} tags=${span_tags.length} ` +
+            `(source=${row.source}, source_id=${row.source_id})`);
+    }
+    // The runner stamps the adapter's `surface` on every row that omits one, so an absent
+    // value here means the row reached parquet by a path that bypassed it.
+    // A default would record every such row as the publisher's own string.
+    if (!row.surface) {
+        throw new Error(`rowToParquet: row carries no surface ` +
+            `(source=${row.source}, source_id=${row.source_id}). ` +
+            `An adapter declares CorpusAdapter.surface and the runner stamps it; ` +
+            `a recipe writing rows directly sets the field itself.`);
+    }
+    return {
+        raw: row.raw,
+        tokens: row.tokens,
+        labels: row.labels,
+        span_starts,
+        span_ends,
+        span_tags,
+        country: row.country,
+        locale: row.locale ?? null,
+        source: row.source,
+        source_id: row.source_id,
+        corpus_version: row.corpus_version,
+        license: row.license,
+        register: row.register ?? null,
+        surface: row.surface,
+        recipe: row.recipe?.recipe ?? null,
+        base_source_id: row.recipe?.base_source_id ?? null,
+    };
+}
+//# sourceMappingURL=schema.js.map

package/out/parquet/schema.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../lib/parquet/schema.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,cAAc,GAAG,MAAM,CAAA;AAEpC;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,SAAS,CAAA;AAEtC;;GAEG;AACH,MAAM,CAAC,MAAM,mBAAmB,GAAG,QAAQ,CAAA;AAuC3C;;;;GAIG;AACH,MAAM,CAAC,MAAM,eAAe,GAAG;IAC9B,KAAK;IACL,QAAQ;IACR,QAAQ;IACR,aAAa;IACb,WAAW;IACX,WAAW;IACX,SAAS;IACT,QAAQ;IACR,QAAQ;IACR,WAAW;IACX,gBAAgB;IAChB,SAAS;IACT,UAAU;IACV,SAAS;IACT,QAAQ;IACR,gBAAgB;CACP,CAAA;AAEV;;;;;GAKG;AACH,MAAM,CAAC,MAAM,oBAAoB,GAAqD;IACrF,GAAG,EAAE,SAAS;IACd,MAAM,EAAE,WAAW;IACnB,MAAM,EAAE,WAAW;IACnB,WAAW,EAAE,WAAW;IACxB,SAAS,EAAE,WAAW;IACtB,SAAS,EAAE,WAAW;IACtB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,MAAM,EAAE,SAAS;IACjB,SAAS,EAAE,SAAS;IACpB,cAAc,EAAE,SAAS;IACzB,OAAO,EAAE,SAAS;IAClB,QAAQ,EAAE,SAAS;IACnB,OAAO,EAAE,SAAS;IAClB,MAAM,EAAE,SAAS;IACjB,cAAc,EAAE,SAAS;CACzB,CAAA;AAED;kFACkF;AAElF;;;;;GAKG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAwC;IACtE,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IACvD,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC1E,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC1E,4QAA4Q;IAC5Q,WAAW,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAChF,SAAS,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC9E,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC7E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC1E,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC1D,SAAS,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC7D,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAClE,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC3D,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC5E,OAAO,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE;IAC3D,MAAM,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE;IAC1E,cAAc,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE;CAClF,CAAA;AAED,yDAAyD;AAEzD;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAAC,GAAe;IAC3C,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,SAAS,EAAE,GAAG,GAAG,CAAA;IAEjD,IAAI,WAAW,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,SAAS,EAAE,CAAC;QACrF,MAAM,IAAI,KAAK,CACd,oEAAoE;YACnE,eAAe,WAAW,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,cAAc,SAAS,KAAK,SAAS,GAAG;YACrH,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,gFAAgF,CACjF,CAAA;IACF,CAAC;IAED,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,IAAI,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxF,MAAM,IAAI,KAAK,CACd,sDAAsD;YACrD,UAAU,WAAW,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,SAAS,SAAS,CAAC,MAAM,GAAG;YACjF,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,GAAG,CACrD,CAAA;IACF,CAAC;IAED,sFAAsF;IACtF,uEAAuE;IACvE,uEAAuE;IACvE,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;QAClB,MAAM,IAAI,KAAK,CACd,uCAAuC;YACtC,WAAW,GAAG,CAAC,MAAM,eAAe,GAAG,CAAC,SAAS,KAAK;YACtD,sEAAsE;YACtE,uDAAuD,CACxD,CAAA;IACF,CAAC;IAED,OAAO;QACN,GAAG,EAAE,GAAG,CAAC,GAAG;QACZ,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,WAAW;QACX,SAAS;QACT,SAAS;QACT,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,IAAI,IAAI;QAC1B,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,SAAS,EAAE,GAAG,CAAC,SAAS;QACxB,cAAc,EAAE,GAAG,CAAC,cAAc;QAClC,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,QAAQ,EAAE,GAAG,CAAC,QAAQ,IAAI,IAAI;QAC9B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,IAAI,IAAI;QAClC,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,IAAI,IAAI;KAClD,CAAA;AACF,CAAC"}

package/out/parquet/shuffle.d.ts

@@ -0,0 +1,25 @@
+/**
+ * @copyright Sister Software
+ * @license AGPL-3.0
+ * @author Teffen Ellis, et al.
+ * @file A bounded streaming shuffle over the rows headed for parquet, so a row-group is a sample of its source.
+ */
+/**
+ * Rows held in memory while shuffling, when the caller names no window.
+ *
+ * Five 50,000-row groups span more than one output row-group and reach `wof-admin`'s
+ * whole 13-country set, while a window below `ROW_GROUP_SIZE` buys no wider mix.
+ */
+export declare const DEFAULT_SHUFFLE_WINDOW = 250000;
+/**
+ * Seed for the shuffle when the caller names none, fixed rather than drawn so two builds
+ * of one input write the same row order and a corpus is reproducible from its inputs.
+ */
+export declare const DEFAULT_SHUFFLE_SEED = 20260922;
+/**
+ * Yield the input shuffled within a sliding window of `windowSize` rows, so memory is
+ * bounded by the window rather than the input; `windowSize <= 1` yields the input unchanged
+ * and consumes no random draw, keeping a no-shuffle build byte-identical.
+ */
+export declare function shuffleWithinWindow<T>(rows: AsyncIterable<T>, random: () => number, windowSize?: number): AsyncGenerator<T>;
+//# sourceMappingURL=shuffle.d.ts.map

package/out/parquet/shuffle.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"shuffle.d.ts","sourceRoot":"","sources":["../../lib/parquet/shuffle.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;;GAKG;AACH,eAAO,MAAM,sBAAsB,SAAU,CAAA;AAE7C;;;GAGG;AACH,eAAO,MAAM,oBAAoB,WAAa,CAAA;AAE9C;;;;GAIG;AACH,wBAAuB,mBAAmB,CAAC,CAAC,EAC3C,IAAI,EAAE,aAAa,CAAC,CAAC,CAAC,EACtB,MAAM,EAAE,MAAM,MAAM,EACpB,UAAU,GAAE,MAA+B,GACzC,cAAc,CAAC,CAAC,CAAC,CA0BnB"}