@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

package/dist/parquetjs/parser/encoded-pages.js

@@ -0,0 +1,206 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { Encoding, PageType } from "../parquet-thrift/index.js";
+import { crc32 } from "../utils/crc32.js";
+import { readUInt32LE } from "../utils/binary-utils.js";
+import { decodePageHeader, getBitWidth, getThriftEnum } from "../utils/read-utils.js";
+/**
+ * Splits a plaintext column-chunk buffer into transport-neutral encoded page descriptors.
+ *
+ * This parses only page headers and level framing. It deliberately does not decode levels,
+ * dictionaries, or values, allowing the returned bytes to be consumed by a CPU or GPU backend.
+ */
+export async function scanParquetEncodedPages(buffer, context) {
+    const pages = [];
+    let offset = 0;
+    let dataPageOrdinal = 0;
+    while (offset < buffer.byteLength) {
+        const { pageHeader, length: headerByteLength } = decodePageHeader(buffer, offset);
+        const bodyOffset = offset + headerByteLength;
+        const bodyEnd = bodyOffset + pageHeader.compressed_page_size;
+        if (pageHeader.compressed_page_size < 0 ||
+            pageHeader.uncompressed_page_size < 0 ||
+            bodyEnd > buffer.byteLength) {
+            throw new Error('Parquet page extends beyond the available column-chunk buffer');
+        }
+        const compressedBody = buffer.subarray(bodyOffset, bodyEnd);
+        verifyEncodedPageChecksum(compressedBody, pageHeader, context.verifyPageChecksums);
+        const pageType = getThriftEnum(PageType, pageHeader.type);
+        if (pageType === 'DICTIONARY_PAGE') {
+            pages.push(await createDictionaryPage(compressedBody, pageHeader, context));
+        }
+        else if (pageType === 'DATA_PAGE') {
+            pages.push(await createDataPageV1(compressedBody, pageHeader, context, dataPageOrdinal++));
+        }
+        else if (pageType === 'DATA_PAGE_V2') {
+            pages.push(await createDataPageV2(compressedBody, pageHeader, context, dataPageOrdinal++));
+        }
+        else {
+            throw new Error(`Unsupported Parquet page type ${pageType}`);
+        }
+        offset = bodyEnd;
+    }
+    return pages;
+}
+/** Creates one dictionary-page descriptor, optionally retaining its compressed bytes. */
+async function createDictionaryPage(compressedBody, pageHeader, context) {
+    const dictionaryHeader = pageHeader.dictionary_page_header;
+    if (!dictionaryHeader) {
+        throw new Error('Parquet dictionary page is missing its dictionary header');
+    }
+    const preserveCompression = shouldPreserveCompression(context);
+    const data = preserveCompression
+        ? compressedBody
+        : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+    return Object.freeze({
+        type: 'dictionary',
+        pageOrdinal: -1,
+        encoding: getThriftEnum(Encoding, dictionaryHeader.encoding),
+        compression: context.compression,
+        compressionState: preserveCompression ? 'compressed' : 'decompressed',
+        valueCount: dictionaryHeader.num_values,
+        nonNullValueCount: dictionaryHeader.num_values,
+        data,
+        values: preserveCompression ? undefined : createSection(0, data.byteLength),
+        compressedByteLength: pageHeader.compressed_page_size,
+        uncompressedByteLength: pageHeader.uncompressed_page_size
+    });
+}
+/** Creates one V1 data-page descriptor and normalizes its length-prefixed level streams. */
+async function createDataPageV1(compressedBody, pageHeader, context, pageOrdinal) {
+    const dataHeader = pageHeader.data_page_header;
+    if (!dataHeader) {
+        throw new Error('Parquet V1 data page is missing its data header');
+    }
+    const preserveCompression = shouldPreserveCompression(context);
+    const data = preserveCompression
+        ? compressedBody
+        : await decompressPageBody(compressedBody, pageHeader.uncompressed_page_size, context);
+    let repetitionLevels;
+    let definitionLevels;
+    let values;
+    if (!preserveCompression) {
+        let sectionOffset = 0;
+        const repetitionLevelEncoding = getThriftEnum(Encoding, dataHeader.repetition_level_encoding);
+        repetitionLevels = readV1LevelSection(data, sectionOffset, dataHeader.num_values, context.column.rLevelMax, repetitionLevelEncoding);
+        sectionOffset = repetitionLevels.byteOffset + repetitionLevels.byteLength;
+        const definitionLevelEncoding = getThriftEnum(Encoding, dataHeader.definition_level_encoding);
+        definitionLevels = readV1LevelSection(data, sectionOffset, dataHeader.num_values, context.column.dLevelMax, definitionLevelEncoding);
+        sectionOffset = definitionLevels.byteOffset + definitionLevels.byteLength;
+        values = createSection(sectionOffset, data.byteLength - sectionOffset);
+    }
+    return Object.freeze({
+        type: 'data-v1',
+        pageOrdinal,
+        encoding: getThriftEnum(Encoding, dataHeader.encoding),
+        repetitionLevelEncoding: getThriftEnum(Encoding, dataHeader.repetition_level_encoding),
+        definitionLevelEncoding: getThriftEnum(Encoding, dataHeader.definition_level_encoding),
+        compression: context.compression,
+        compressionState: preserveCompression ? 'compressed' : 'decompressed',
+        valueCount: dataHeader.num_values,
+        data,
+        repetitionLevels,
+        definitionLevels,
+        values,
+        compressedByteLength: pageHeader.compressed_page_size,
+        uncompressedByteLength: pageHeader.uncompressed_page_size
+    });
+}
+/** Creates one V2 data-page descriptor whose level sections are already length-delimited. */
+async function createDataPageV2(compressedBody, pageHeader, context, pageOrdinal) {
+    const dataHeader = pageHeader.data_page_header_v2;
+    if (!dataHeader) {
+        throw new Error('Parquet V2 data page is missing its data header');
+    }
+    const repetitionByteLength = dataHeader.repetition_levels_byte_length;
+    const definitionByteLength = dataHeader.definition_levels_byte_length;
+    const levelsByteLength = repetitionByteLength + definitionByteLength;
+    if (levelsByteLength > compressedBody.byteLength) {
+        throw new Error('Parquet V2 level sections extend beyond the page body');
+    }
+    const valuesAreCompressed = dataHeader.is_compressed !== false && context.compression !== 'UNCOMPRESSED';
+    const preserveCompression = valuesAreCompressed && shouldPreserveCompression(context);
+    let data = compressedBody;
+    if (valuesAreCompressed && !preserveCompression) {
+        const uncompressedValuesByteLength = pageHeader.uncompressed_page_size - levelsByteLength;
+        if (uncompressedValuesByteLength < 0) {
+            throw new Error('Invalid Parquet V2 uncompressed value length');
+        }
+        const values = await context.decompressPage(compressedBody.subarray(levelsByteLength), uncompressedValuesByteLength);
+        data = new Uint8Array(levelsByteLength + values.byteLength);
+        data.set(compressedBody.subarray(0, levelsByteLength));
+        data.set(values, levelsByteLength);
+    }
+    return Object.freeze({
+        type: 'data-v2',
+        pageOrdinal,
+        encoding: getThriftEnum(Encoding, dataHeader.encoding),
+        repetitionLevelEncoding: 'RLE',
+        definitionLevelEncoding: 'RLE',
+        compression: context.compression,
+        compressionState: preserveCompression ? 'compressed' : 'decompressed',
+        valueCount: dataHeader.num_values,
+        nonNullValueCount: dataHeader.num_values - dataHeader.num_nulls,
+        data,
+        repetitionLevels: createSection(0, repetitionByteLength),
+        definitionLevels: createSection(repetitionByteLength, definitionByteLength),
+        values: createSection(levelsByteLength, data.byteLength - levelsByteLength),
+        compressedByteLength: pageHeader.compressed_page_size,
+        uncompressedByteLength: pageHeader.uncompressed_page_size
+    });
+}
+/** Returns whether the downstream decoder requested this nontrivial codec verbatim. */
+function shouldPreserveCompression(context) {
+    return (context.compression !== 'UNCOMPRESSED' &&
+        Boolean(context.preserveCompression?.has(context.compression)));
+}
+/** Inflates a complete V1 or dictionary page body when its codec is not deferred. */
+async function decompressPageBody(compressedBody, uncompressedByteLength, context) {
+    if (context.compression === 'UNCOMPRESSED')
+        return compressedBody;
+    return await context.decompressPage(compressedBody, uncompressedByteLength);
+}
+/** Locates one V1 repetition/definition level stream without decoding its level values. */
+function readV1LevelSection(data, offset, valueCount, maxLevel, encoding) {
+    if (maxLevel === 0)
+        return createSection(offset, 0);
+    if (encoding === 'RLE') {
+        if (offset + 4 > data.byteLength) {
+            throw new Error('Parquet V1 RLE level stream is missing its length prefix');
+        }
+        const byteLength = readUInt32LE(data, offset);
+        const byteOffset = offset + 4;
+        assertSectionBounds(data, byteOffset, byteLength);
+        return createSection(byteOffset, byteLength);
+    }
+    if (encoding === 'BIT_PACKED') {
+        // Legacy BIT_PACKED levels are padded to complete groups of eight values. This differs from
+        // simply rounding the total bit count and matters whenever bit width exceeds one.
+        const byteLength = Math.ceil(valueCount / 8) * getBitWidth(maxLevel);
+        assertSectionBounds(data, offset, byteLength);
+        return createSection(offset, byteLength);
+    }
+    throw new Error(`Unsupported Parquet V1 level encoding ${encoding}`);
+}
+/** Creates an immutable section descriptor. */
+function createSection(byteOffset, byteLength) {
+    return Object.freeze({ byteOffset, byteLength });
+}
+/** Validates a page section before exposing it to a deferred decoder. */
+function assertSectionBounds(data, byteOffset, byteLength) {
+    if (byteOffset < 0 || byteLength < 0 || byteOffset + byteLength > data.byteLength) {
+        throw new Error('Parquet page section extends beyond the page body');
+    }
+}
+/** Verifies a page checksum against its stored, still-compressed body. */
+function verifyEncodedPageChecksum(compressedBody, pageHeader, verifyPageChecksums = false) {
+    if (!verifyPageChecksums || pageHeader.crc === undefined)
+        return;
+    const expected = pageHeader.crc >>> 0;
+    const actual = crc32(compressedBody);
+    if (actual !== expected) {
+        throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
+    }
+}
+//# sourceMappingURL=encoded-pages.js.map

package/dist/parquetjs/parser/encoded-pages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encoded-pages.js","sourceRoot":"","sources":["../../../src/parquetjs/parser/encoded-pages.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAIpC,OAAO,EAAC,QAAQ,EAAE,QAAQ,EAAkB,mCAAgC;AAC5E,OAAO,EAAC,KAAK,EAAC,0BAAuB;AACrC,OAAO,EAAC,YAAY,EAAC,iCAA8B;AACnD,OAAO,EAAC,gBAAgB,EAAE,WAAW,EAAE,aAAa,EAAC,+BAA4B;AAiBjF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,MAAkB,EAClB,OAAkC;IAElC,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,OAAO,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,EAAC,UAAU,EAAE,MAAM,EAAE,gBAAgB,EAAC,GAAG,gBAAgB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAChF,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAgB,CAAC;QAC7C,MAAM,OAAO,GAAG,UAAU,GAAG,UAAU,CAAC,oBAAoB,CAAC;QAC7D,IACE,UAAU,CAAC,oBAAoB,GAAG,CAAC;YACnC,UAAU,CAAC,sBAAsB,GAAG,CAAC;YACrC,OAAO,GAAG,MAAM,CAAC,UAAU,EAC3B,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,+DAA+D,CAAC,CAAC;QACnF,CAAC;QACD,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC5D,yBAAyB,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,mBAAmB,CAAC,CAAC;QACnF,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;QAC1D,IAAI,QAAQ,KAAK,iBAAiB,EAAE,CAAC;YACnC,KAAK,CAAC,IAAI,CAAC,MAAM,oBAAoB,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC;QAC9E,CAAC;aAAM,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC;aAAM,IAAI,QAAQ,KAAK,cAAc,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,MAAM,gBAAgB,CAAC,cAAc,EAAE,UAAU,EAAE,OAAO,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,iCAAiC,QAAQ,EAAE,CAAC,CAAC;QAC/D,CAAC;QACD,MAAM,GAAG,OAAO,CAAC;IACnB,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,yFAAyF;AACzF,KAAK,UAAU,oBAAoB,CACjC,cAA0B,EAC1B,UAAsB,EACtB,OAAkC;IAElC,MAAM,gBAAgB,GAAG,UAAU,CAAC,sBAAsB,CAAC;IAC3D,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,MAAM,mBAAmB,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,mBAAmB;QAC9B,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,MAAM,kBAAkB,CAAC,cAAc,EAAE,UAAU,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACzF,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,YAAqB;QAC3B,WAAW,EAAE,CAAC,CAAC;QACf,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,gBAAgB,CAAC,QAAQ,CAAC;QAC5D,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAE,YAAsB,CAAC,CAAC,CAAE,cAAwB;QAC3F,UAAU,EAAE,gBAAgB,CAAC,UAAU;QACvC,iBAAiB,EAAE,gBAAgB,CAAC,UAAU;QAC9C,IAAI;QACJ,MAAM,EAAE,mBAAmB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC;QAC3E,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;QACrD,sBAAsB,EAAE,UAAU,CAAC,sBAAsB;KAC1D,CAAC,CAAC;AACL,CAAC;AAED,4FAA4F;AAC5F,KAAK,UAAU,gBAAgB,CAC7B,cAA0B,EAC1B,UAAsB,EACtB,OAAkC,EAClC,WAAmB;IAEnB,MAAM,UAAU,GAAG,UAAU,CAAC,gBAAgB,CAAC;IAC/C,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,mBAAmB,GAAG,yBAAyB,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,mBAAmB;QAC9B,CAAC,CAAC,cAAc;QAChB,CAAC,CAAC,MAAM,kBAAkB,CAAC,cAAc,EAAE,UAAU,CAAC,sBAAsB,EAAE,OAAO,CAAC,CAAC;IACzF,IAAI,gBAAuD,CAAC;IAC5D,IAAI,gBAAuD,CAAC;IAC5D,IAAI,MAA6C,CAAC;IAClD,IAAI,CAAC,mBAAmB,EAAE,CAAC;QACzB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,MAAM,uBAAuB,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAC9F,gBAAgB,GAAG,kBAAkB,CACnC,IAAI,EACJ,aAAa,EACb,UAAU,CAAC,UAAU,EACrB,OAAO,CAAC,MAAM,CAAC,SAAS,EACxB,uBAAuB,CACxB,CAAC;QACF,aAAa,GAAG,gBAAgB,CAAC,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC;QAC1E,MAAM,uBAAuB,GAAG,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC,CAAC;QAC9F,gBAAgB,GAAG,kBAAkB,CACnC,IAAI,EACJ,aAAa,EACb,UAAU,CAAC,UAAU,EACrB,OAAO,CAAC,MAAM,CAAC,SAAS,EACxB,uBAAuB,CACxB,CAAC;QACF,aAAa,GAAG,gBAAgB,CAAC,UAAU,GAAG,gBAAgB,CAAC,UAAU,CAAC;QAC1E,MAAM,GAAG,aAAa,CAAC,aAAa,EAAE,IAAI,CAAC,UAAU,GAAG,aAAa,CAAC,CAAC;IACzE,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,SAAkB;QACxB,WAAW;QACX,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;QACtD,uBAAuB,EAAE,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC;QACtF,uBAAuB,EAAE,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,yBAAyB,CAAC;QACtF,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAE,YAAsB,CAAC,CAAC,CAAE,cAAwB;QAC3F,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,IAAI;QACJ,gBAAgB;QAChB,gBAAgB;QAChB,MAAM;QACN,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;QACrD,sBAAsB,EAAE,UAAU,CAAC,sBAAsB;KAC1D,CAAC,CAAC;AACL,CAAC;AAED,6FAA6F;AAC7F,KAAK,UAAU,gBAAgB,CAC7B,cAA0B,EAC1B,UAAsB,EACtB,OAAkC,EAClC,WAAmB;IAEnB,MAAM,UAAU,GAAG,UAAU,CAAC,mBAAmB,CAAC;IAClD,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,MAAM,oBAAoB,GAAG,UAAU,CAAC,6BAA6B,CAAC;IACtE,MAAM,oBAAoB,GAAG,UAAU,CAAC,6BAA6B,CAAC;IACtE,MAAM,gBAAgB,GAAG,oBAAoB,GAAG,oBAAoB,CAAC;IACrE,IAAI,gBAAgB,GAAG,cAAc,CAAC,UAAU,EAAE,CAAC;QACjD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,mBAAmB,GACvB,UAAU,CAAC,aAAa,KAAK,KAAK,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc,CAAC;IAC/E,MAAM,mBAAmB,GAAG,mBAAmB,IAAI,yBAAyB,CAAC,OAAO,CAAC,CAAC;IACtF,IAAI,IAAI,GAAG,cAAc,CAAC;IAC1B,IAAI,mBAAmB,IAAI,CAAC,mBAAmB,EAAE,CAAC;QAChD,MAAM,4BAA4B,GAAG,UAAU,CAAC,sBAAsB,GAAG,gBAAgB,CAAC;QAC1F,IAAI,4BAA4B,GAAG,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CACzC,cAAc,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EACzC,4BAA4B,CAC7B,CAAC;QACF,IAAI,GAAG,IAAI,UAAU,CAAC,gBAAgB,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAC5D,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC,EAAE,gBAAgB,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC;QACnB,IAAI,EAAE,SAAkB;QACxB,WAAW;QACX,QAAQ,EAAE,aAAa,CAAC,QAAQ,EAAE,UAAU,CAAC,QAAQ,CAAC;QACtD,uBAAuB,EAAE,KAAK;QAC9B,uBAAuB,EAAE,KAAK;QAC9B,WAAW,EAAE,OAAO,CAAC,WAAW;QAChC,gBAAgB,EAAE,mBAAmB,CAAC,CAAC,CAAE,YAAsB,CAAC,CAAC,CAAE,cAAwB;QAC3F,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,iBAAiB,EAAE,UAAU,CAAC,UAAU,GAAG,UAAU,CAAC,SAAS;QAC/D,IAAI;QACJ,gBAAgB,EAAE,aAAa,CAAC,CAAC,EAAE,oBAAoB,CAAC;QACxD,gBAAgB,EAAE,aAAa,CAAC,oBAAoB,EAAE,oBAAoB,CAAC;QAC3E,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,IAAI,CAAC,UAAU,GAAG,gBAAgB,CAAC;QAC3E,oBAAoB,EAAE,UAAU,CAAC,oBAAoB;QACrD,sBAAsB,EAAE,UAAU,CAAC,sBAAsB;KAC1D,CAAC,CAAC;AACL,CAAC;AAED,uFAAuF;AACvF,SAAS,yBAAyB,CAAC,OAAkC;IACnE,OAAO,CACL,OAAO,CAAC,WAAW,KAAK,cAAc;QACtC,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAC/D,CAAC;AACJ,CAAC;AAED,qFAAqF;AACrF,KAAK,UAAU,kBAAkB,CAC/B,cAA0B,EAC1B,sBAA8B,EAC9B,OAAkC;IAElC,IAAI,OAAO,CAAC,WAAW,KAAK,cAAc;QAAE,OAAO,cAAc,CAAC;IAClE,OAAO,MAAM,OAAO,CAAC,cAAc,CAAC,cAAc,EAAE,sBAAsB,CAAC,CAAC;AAC9E,CAAC;AAED,2FAA2F;AAC3F,SAAS,kBAAkB,CACzB,IAAgB,EAChB,MAAc,EACd,UAAkB,EAClB,QAAgB,EAChB,QAAgB;IAEhB,IAAI,QAAQ,KAAK,CAAC;QAAE,OAAO,aAAa,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpD,IAAI,QAAQ,KAAK,KAAK,EAAE,CAAC;QACvB,IAAI,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;QAC9E,CAAC;QACD,MAAM,UAAU,GAAG,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9C,MAAM,UAAU,GAAG,MAAM,GAAG,CAAC,CAAC;QAC9B,mBAAmB,CAAC,IAAI,EAAE,UAAU,EAAE,UAAU,CAAC,CAAC;QAClD,OAAO,aAAa,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,QAAQ,KAAK,YAAY,EAAE,CAAC;QAC9B,4FAA4F;QAC5F,kFAAkF;QAClF,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,WAAW,CAAC,QAAQ,CAAC,CAAC;QACrE,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;QAC9C,OAAO,aAAa,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IAC3C,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,QAAQ,EAAE,CAAC,CAAC;AACvE,CAAC;AAED,+CAA+C;AAC/C,SAAS,aAAa,CAAC,UAAkB,EAAE,UAAkB;IAC3D,OAAO,MAAM,CAAC,MAAM,CAAC,EAAC,UAAU,EAAE,UAAU,EAAC,CAAC,CAAC;AACjD,CAAC;AAED,yEAAyE;AACzE,SAAS,mBAAmB,CAAC,IAAgB,EAAE,UAAkB,EAAE,UAAkB;IACnF,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,IAAI,UAAU,GAAG,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;QAClF,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;AACH,CAAC;AAED,0EAA0E;AAC1E,SAAS,yBAAyB,CAChC,cAA0B,EAC1B,UAAsB,EACtB,mBAAmB,GAAG,KAAK;IAE3B,IAAI,CAAC,mBAAmB,IAAI,UAAU,CAAC,GAAG,KAAK,SAAS;QAAE,OAAO;IACjE,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,KAAK,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;IACrC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,4CAA4C,QAAQ,gBAAgB,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;AACH,CAAC"}

package/dist/parquetjs/parser/parquet-reader.d.ts

@@ -1,8 +1,10 @@
 import type { ReadableFile } from '@loaders.gl/loader-utils';
 import { ParquetSchema } from "../schema/schema.js";
+import type { ParquetEncodedColumnChunk } from "../../parquet-encoded-page-types.js";
 import { ColumnChunk, FileMetaData, RowGroup } from "../parquet-thrift/index.js";
 import { ParquetRowGroup, ParquetColumnChunk, ParquetReaderContext } from "../schema/declare.js";
 import type { ParquetDataPageLocation, ParquetPageLocations, ParquetRowRange } from "../../lib/parquet-page-index.js";
+import { type ParquetEncryptionAlgorithm, type ParquetKeyRetriever } from "../../lib/parquet-encryption.js";
 export type ParquetReaderProps = {
     /** Maximum dictionary-page read size. */
     defaultDictionarySize?: number;
@@ -14,8 +16,22 @@ export type ParquetReaderProps = {
     useTypedValueBuffers?: boolean;
     /** Decode repetition and definition levels into compact unsigned typed arrays. */
     useTypedLevelBuffers?: boolean;
+    /** Decode eligible PLAIN BYTE_ARRAY columns into Arrow-compatible contiguous buffers. */
+    useArrowByteArrayBuffers?: boolean;
+    /** Verify page-header CRC values when present. Disabled by default for throughput. */
+    verifyPageChecksums?: boolean;
+    /** Decode legacy INT96 values as epoch nanoseconds for Arrow timestamp columns. */
+    int96AsTimestamp?: boolean;
+    /** Verify plaintext-footer signatures when present. Enabled by default. */
+    verifyFooterSignature?: boolean;
     /** Abort signal forwarded to every underlying random-access read. */
     signal?: AbortSignal;
+    /** Resolves modular-encryption keys from the file's key metadata. */
+    keyRetriever?: ParquetKeyRetriever;
+    /** AAD prefix for files that intentionally omit it from FileCryptoMetaData. */
+    aadPrefix?: Uint8Array;
+    /** Pre-resolved encryption context used by worker-local readers. */
+    encryptionContext?: ParquetReaderEncryptionContext;
 };
 /** Properties for initializing a ParquetRowGroupReader */
 export type ParquetIterationProps = {
@@ -26,6 +42,17 @@ export type ParquetIterationProps = {
     /** Abort signal forwarded to row-group and column-chunk reads. */
     signal?: AbortSignal;
 };
+type NormalizedParquetReaderProps = Required<Omit<ParquetReaderProps, 'signal' | 'keyRetriever' | 'aadPrefix' | 'encryptionContext'>> & Pick<ParquetReaderProps, 'signal' | 'keyRetriever' | 'aadPrefix' | 'encryptionContext'>;
+/** File-level parameters needed to authenticate modular-encrypted modules. */
+type ParquetEncryptionContext = {
+    algorithm: ParquetEncryptionAlgorithm;
+    aadPrefix?: Uint8Array;
+    fileUnique: Uint8Array;
+    footerKeyMetadata?: Uint8Array;
+    footerSigningKeyMetadata?: Uint8Array;
+};
+/** File-level encryption parameters needed to decrypt worker-transferred page modules. */
+export type ParquetReaderEncryptionContext = Pick<ParquetEncryptionContext, 'algorithm' | 'aadPrefix' | 'fileUnique' | 'footerKeyMetadata'>;
 /**
  * The parquet envelope reader allows direct, unbuffered access to the individual
  * sections of the parquet file, namely the header, footer and the row groups.
@@ -33,18 +60,37 @@ export type ParquetIterationProps = {
  * rows from a parquet file use the ParquetReader instead
  */
 export declare class ParquetReader {
-    static defaultProps: Required<Omit<ParquetReaderProps, 'signal'>> & {
-        signal?: AbortSignal;
-    };
-    props: Required<Omit<ParquetReaderProps, 'signal'>> & {
-        signal?: AbortSignal;
-    };
+    static defaultProps: NormalizedParquetReaderProps;
+    props: NormalizedParquetReaderProps;
     file: ReadableFile;
     metadata: Promise<FileMetaData> | null;
     /** Parsed Parquet schema shared by metadata, iteration, and materialization paths. */
     private schema;
+    /** Encryption parameters decoded from plaintext or encrypted footer metadata. */
+    private encryptionContext?;
+    /** Resolved keys shared by encrypted modules in this reader instance. */
+    private readonly encryptionKeyCache;
+    /** Compression backends selected once and shared by every page in this reader. */
+    private readonly pageDecompressors;
     constructor(file: ReadableFile, props?: ParquetReaderProps);
+    /** Whether the decoded footer describes an encrypted Parquet file. */
+    get encrypted(): boolean;
+    /** Whether this reader decodes legacy INT96 values as epoch nanoseconds. */
+    get int96AsTimestamp(): boolean;
+    /** Returns a cloneable snapshot of the file encryption context for worker decoding. */
+    getEncryptionContextForWorker(): ParquetReaderEncryptionContext | undefined;
+    /** Resolves one column key on the caller thread before transferring encrypted ranges. */
+    getColumnKeyForWorker(columnChunk: ColumnChunk, rowGroupOrdinal: number, columnOrdinal: number): Promise<{
+        keyMetadata?: Uint8Array;
+        keyMaterial: Uint8Array;
+    }>;
     close(): void;
+    /** Returns the reader-scoped decoder for one Parquet compression method. */
+    private getPageDecompressor;
+    /** Reads a byte range, retaining a zero-copy view when the complete file is already in memory. */
+    private readBytes;
+    /** Returns a reader-scoped key resolver that avoids repeating key-provider work per module. */
+    private getCachedKeyRetriever;
     /** Yield one row at a time */
     rowIterator(props?: ParquetIterationProps): AsyncGenerator<import("../schema/declare").ParquetRow, void, unknown>;
     /** Yield one batch of rows at a time */
@@ -63,16 +109,45 @@ export declare class ParquetReader {
     readHeader(signal?: AbortSignal): Promise<void>;
     /** Metadata is stored in the footer */
     readFooter(signal?: AbortSignal): Promise<FileMetaData>;
+    /** Reads and decrypts an encrypted footer after the standard trailer. */
+    private readEncryptedFooter;
+    /** Records the encryption parameters exposed by plaintext-footer files. */
+    private setEncryptionContext;
+    /** Resolves only the selected encrypted columns, using crypto metadata paths before decryption. */
+    resolveColumnMetadata(rowGroup: RowGroup, rowGroupOrdinal: number, selectedColumnPaths?: readonly string[][]): Promise<void>;
+    /** Decrypts one encrypted column index or offset index module for selective planning. */
+    decryptIndexModule(bytes: Uint8Array, module: 'column-index' | 'offset-index', rowGroupOrdinal: number, columnOrdinal: number, columnChunk: ColumnChunk): Promise<Uint8Array>;
+    /** Decrypts the encrypted Bloom-filter header and bitset modules for one column chunk. */
+    decryptBloomFilter(bytes: Uint8Array, rowGroupOrdinal: number, columnOrdinal: number, columnChunk: ColumnChunk): Promise<Uint8Array>;
+    /** Resolves a column's plaintext metadata, decrypting column-specific metadata when present. */
+    private getColumnMetadata;
     /** Data is stored in row groups (similar to Apache Arrow record batches) */
-    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], signal?: AbortSignal): Promise<ParquetRowGroup>;
-    /** Reads independently materializable flat columns for one logical row range using page indexes. */
-    readRowGroupRange(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], rowRange: ParquetRowRange, pageLocations: ParquetPageLocations, signal?: AbortSignal): Promise<ParquetRowGroup>;
+    readRowGroup(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], signal?: AbortSignal, rowGroupOrdinal?: number): Promise<ParquetRowGroup>;
+    /**
+     * Reads selected column chunks as encoded page descriptors without materializing values.
+     *
+     * Page headers, encryption, checksums, and optional decompression remain reader concerns. Value,
+     * dictionary, and level decoding is deliberately left to the downstream consumer.
+     */
+    readRowGroupEncodedPages(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], preserveCompression: ReadonlySet<string>, signal?: AbortSignal, rowGroupOrdinal?: number): Promise<readonly ParquetEncodedColumnChunk[]>;
+    /** Reads independently materializable non-repeated columns for one row range using page indexes. */
+    readRowGroupRange(schema: ParquetSchema, rowGroup: RowGroup, columnList: string[][], rowRange: ParquetRowRange, pageLocations: ParquetPageLocations, signal?: AbortSignal, rowGroupOrdinal?: number): Promise<ParquetRowGroup>;
+    /**
+     * Each row group contains column chunks for all the columns.
+     */
+    private readUncompressedColumnChunkFromMemory;
     /**
      * Each row group contains column chunks for all the columns.
      */
-    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk, signal?: AbortSignal): Promise<ParquetColumnChunk>;
-    /** Reads and decodes only the contiguous data pages overlapping one flat-column row range. */
-    readColumnChunkRange(schema: ParquetSchema, columnChunk: ColumnChunk, rowRange: ParquetRowRange, pages: readonly ParquetDataPageLocation[], signal?: AbortSignal): Promise<ParquetColumnChunk>;
+    readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk, signal?: AbortSignal, rowGroupOrdinal?: number, columnOrdinal?: number, resolvedMetadata?: NonNullable<ColumnChunk['meta_data']>): Promise<ParquetColumnChunk>;
+    /** Reads one column chunk as encoded dictionaries, levels, and values for deferred decoding. */
+    readColumnChunkEncodedPages(schema: ParquetSchema, columnChunk: ColumnChunk, preserveCompression: ReadonlySet<string>, signal?: AbortSignal, rowGroupOrdinal?: number, columnOrdinal?: number, resolvedMetadata?: NonNullable<ColumnChunk['meta_data']>): Promise<ParquetEncodedColumnChunk>;
+    /** Decrypts the length-prefixed page header and data modules in one column chunk. */
+    private decryptColumnPages;
+    /** Resolves the key metadata reference for an encrypted column. */
+    private getColumnKeyMetadata;
+    /** Reads and decodes contiguous data pages that begin and end on complete row boundaries. */
+    readColumnChunkRange(schema: ParquetSchema, columnChunk: ColumnChunk, rowRange: ParquetRowRange, pages: readonly ParquetDataPageLocation[], signal?: AbortSignal, rowGroupOrdinal?: number, columnOrdinal?: number, resolvedMetadata?: NonNullable<ColumnChunk['meta_data']>): Promise<ParquetColumnChunk>;
     /**
      * Getting dictionary for allows to flatten values by indices.
      * @param dictionaryPageOffset
@@ -82,4 +157,5 @@ export declare class ParquetReader {
      */
     getDictionary(dictionaryPageOffset: number, context: ParquetReaderContext, pagesOffset: number, signal?: AbortSignal): Promise<any[]>;
 }
+export {};
 //# sourceMappingURL=parquet-reader.d.ts.map

package/dist/parquetjs/parser/parquet-reader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAK/C,OAAO,EAAC,WAAW,EAAoB,YAAY,EAAE,QAAQ,EAAO,mCAAgC;AACpG,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EAErB,6BAA0B;AAI3B,OAAO,KAAK,EACV,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EAChB,wCAAqC;AAKtC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,yCAAyC;IACzC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wFAAwF;IACxF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,gGAAgG;IAChG,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,qEAAqE;IACrE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;IACnC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,GAAG;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAC,CAQxF;IAEF,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC,kBAAkB,EAAE,QAAQ,CAAC,CAAC,GAAG;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAC,CAAC;IAC7E,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;IAC9C,sFAAsF;IACtF,OAAO,CAAC,MAAM,CAAuC;gBAEzC,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAK1D,KAAK,IAAI,IAAI;IAOb,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAyB/C,WAAW,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD,SAAS,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;IAS7D;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAUlE,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAarD,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAiC7D,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC;IA+B3B,oGAAoG;IAC9F,iBAAiB,CACrB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,QAAQ,EAAE,eAAe,EACzB,aAAa,EAAE,oBAAoB,EACnC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,eAAe,CAAC;IA4B3B;;OAEG;IACG,eAAe,CACnB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,WAAW,EACrB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC;IAsE9B,8FAA8F;IACxF,oBAAoB,CACxB,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,SAAS,uBAAuB,EAAE,EACzC,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,kBAAkB,CAAC;IA0D9B;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,EAAE,CAAC;CAyBlB"}
+{"version":3,"file":"parquet-reader.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/parser/parquet-reader.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAE3D,OAAO,EAAC,aAAa,EAAC,4BAAyB;AAC/C,OAAO,KAAK,EAAC,yBAAyB,EAAC,4CAAyC;AAahF,OAAO,EACL,WAAW,EAGX,YAAY,EAEZ,QAAQ,EAET,mCAAgC;AACjC,OAAO,EACL,eAAe,EAEf,kBAAkB,EAElB,oBAAoB,EAErB,6BAA0B;AAY3B,OAAO,KAAK,EACV,uBAAuB,EACvB,oBAAoB,EACpB,eAAe,EAChB,wCAAqC;AACtC,OAAO,EAKL,KAAK,0BAA0B,EAC/B,KAAK,mBAAmB,EACzB,wCAAqC;AA0CtC,MAAM,MAAM,kBAAkB,GAAG;IAC/B,yCAAyC;IACzC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,wFAAwF;IACxF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,gGAAgG;IAChG,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,kFAAkF;IAClF,oBAAoB,CAAC,EAAE,OAAO,CAAC;IAC/B,yFAAyF;IACzF,wBAAwB,CAAC,EAAE,OAAO,CAAC;IACnC,sFAAsF;IACtF,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,mFAAmF;IACnF,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,2EAA2E;IAC3E,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,qEAAqE;IACrE,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,qEAAqE;IACrE,YAAY,CAAC,EAAE,mBAAmB,CAAC;IACnC,+EAA+E;IAC/E,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,8BAA8B,CAAC;CACpD,CAAC;AAEF,0DAA0D;AAC1D,MAAM,MAAM,qBAAqB,GAAG;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,EAAE,CAAC;IACnC,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;IACrB,kEAAkE;IAClE,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,KAAK,4BAA4B,GAAG,QAAQ,CAC1C,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,mBAAmB,CAAC,CACxF,GACC,IAAI,CAAC,kBAAkB,EAAE,QAAQ,GAAG,cAAc,GAAG,WAAW,GAAG,mBAAmB,CAAC,CAAC;AAE1F,8EAA8E;AAC9E,KAAK,wBAAwB,GAAG;IAC9B,SAAS,EAAE,0BAA0B,CAAC;IACtC,SAAS,CAAC,EAAE,UAAU,CAAC;IACvB,UAAU,EAAE,UAAU,CAAC;IACvB,iBAAiB,CAAC,EAAE,UAAU,CAAC;IAC/B,wBAAwB,CAAC,EAAE,UAAU,CAAC;CACvC,CAAC;AAEF,0FAA0F;AAC1F,MAAM,MAAM,8BAA8B,GAAG,IAAI,CAC/C,wBAAwB,EACxB,WAAW,GAAG,WAAW,GAAG,YAAY,GAAG,mBAAmB,CAC/D,CAAC;AAEF;;;;;GAKG;AACH,qBAAa,aAAa;IACxB,MAAM,CAAC,YAAY,EAAE,4BAA4B,CAe/C;IAEF,KAAK,EAAE,4BAA4B,CAAC;IACpC,IAAI,EAAE,YAAY,CAAC;IACnB,QAAQ,EAAE,OAAO,CAAC,YAAY,CAAC,GAAG,IAAI,CAAQ;IAC9C,sFAAsF;IACtF,OAAO,CAAC,MAAM,CAAuC;IACrD,iFAAiF;IACjF,OAAO,CAAC,iBAAiB,CAAC,CAA2B;IACrD,yEAAyE;IACzE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAA6D;IAChG,kFAAkF;IAClF,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAA0D;gBAEhF,IAAI,EAAE,YAAY,EAAE,KAAK,CAAC,EAAE,kBAAkB;IAM1D,sEAAsE;IACtE,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED,4EAA4E;IAC5E,IAAI,gBAAgB,IAAI,OAAO,CAE9B;IAED,uFAAuF;IACvF,6BAA6B,IAAI,8BAA8B,GAAG,SAAS;IAY3E,yFAAyF;IACnF,qBAAqB,CACzB,WAAW,EAAE,WAAW,EACxB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,GACpB,OAAO,CAAC;QAAC,WAAW,CAAC,EAAE,UAAU,CAAC;QAAC,WAAW,EAAE,UAAU,CAAA;KAAC,CAAC;IAuB/D,KAAK,IAAI,IAAI;IAOb,4EAA4E;IAC5E,OAAO,CAAC,mBAAmB;IAS3B,kGAAkG;YACpF,SAAS;IAevB,+FAA+F;IAC/F,OAAO,CAAC,qBAAqB;IA8B7B,8BAA8B;IACvB,WAAW,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAShD,wCAAwC;IACjC,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IAOrD,sCAAsC;IAC/B,gBAAgB,CAAC,KAAK,CAAC,EAAE,qBAAqB;IA0B/C,WAAW,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;IAKlD,SAAS,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC;IAS7D;;;OAGG;IACG,iBAAiB,IAAI,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IASpD,eAAe,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAUlE,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAYrD,uCAAuC;IACjC,UAAU,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAqD7D,yEAAyE;YAC3D,mBAAmB;IAmCjC,2EAA2E;IAC3E,OAAO,CAAC,oBAAoB;IAgB5B,mGAAmG;IAC7F,qBAAqB,CACzB,QAAQ,EAAE,QAAQ,EAClB,eAAe,EAAE,MAAM,EACvB,mBAAmB,CAAC,EAAE,SAAS,MAAM,EAAE,EAAE,GACxC,OAAO,CAAC,IAAI,CAAC;IAuBhB,yFAAyF;IACnF,kBAAkB,CACtB,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,cAAc,GAAG,cAAc,EACvC,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,UAAU,CAAC;IAqBtB,0FAA0F;IACpF,kBAAkB,CACtB,KAAK,EAAE,UAAU,EACjB,eAAe,EAAE,MAAM,EACvB,aAAa,EAAE,MAAM,EACrB,WAAW,EAAE,WAAW,GACvB,OAAO,CAAC,UAAU,CAAC;IAuCtB,gGAAgG;YAClF,iBAAiB;IAiD/B,4EAA4E;IACtE,YAAY,CAChB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,GAClB,OAAO,CAAC,eAAe,CAAC;IAmF3B;;;;;OAKG;IACG,wBAAwB,CAC5B,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,GAClB,OAAO,CAAC,SAAS,yBAAyB,EAAE,CAAC;IAmDhD,oGAAoG;IAC9F,iBAAiB,CACrB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,QAAQ,EAClB,UAAU,EAAE,MAAM,EAAE,EAAE,EACtB,QAAQ,EAAE,eAAe,EACzB,aAAa,EAAE,oBAAoB,EACnC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,GAClB,OAAO,CAAC,eAAe,CAAC;IA+C3B;;OAEG;IACH,OAAO,CAAC,qCAAqC;IAuE7C;;OAEG;IACG,eAAe,CACnB,MAAM,EAAE,aAAa,EACrB,QAAQ,EAAE,WAAW,EACrB,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,EACnB,aAAa,SAAI,EACjB,gBAAgB,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,GACvD,OAAO,CAAC,kBAAkB,CAAC;IAkG9B,gGAAgG;IAC1F,2BAA2B,CAC/B,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,WAAW,EACxB,mBAAmB,EAAE,WAAW,CAAC,MAAM,CAAC,EACxC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,EACnB,aAAa,SAAI,EACjB,gBAAgB,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,GACvD,OAAO,CAAC,yBAAyB,CAAC;IAsGrC,qFAAqF;YACvE,kBAAkB;IAsEhC,mEAAmE;IACnE,OAAO,CAAC,oBAAoB;IAQ5B,6FAA6F;IACvF,oBAAoB,CACxB,MAAM,EAAE,aAAa,EACrB,WAAW,EAAE,WAAW,EACxB,QAAQ,EAAE,eAAe,EACzB,KAAK,EAAE,SAAS,uBAAuB,EAAE,EACzC,MAAM,CAAC,EAAE,WAAW,EACpB,eAAe,SAAI,EACnB,aAAa,SAAI,EACjB,gBAAgB,CAAC,EAAE,WAAW,CAAC,WAAW,CAAC,WAAW,CAAC,CAAC,GACvD,OAAO,CAAC,kBAAkB,CAAC;IA0H9B;;;;;;OAMG;IACG,aAAa,CACjB,oBAAoB,EAAE,MAAM,EAC5B,OAAO,EAAE,oBAAoB,EAC7B,WAAW,EAAE,MAAM,EACnB,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,GAAG,EAAE,CAAC;CAoBlB"}