@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/dist/parquetjs/parser/decoders.js

@@ -4,10 +4,13 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 import { PARQUET_CODECS } from "../codecs/index.js";
-import { ConvertedType, Encoding, FieldRepetitionType, PageType, Type } from "../parquet-thrift/index.js";
+import { ConvertedType, EdgeInterpolationAlgorithm, Encoding, FieldRepetitionType, PageType, Type } from "../parquet-thrift/index.js";
 import { decompress } from "../compression.js";
 import { PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING } from "../../lib/constants.js";
 import { decodePageHeader, getThriftEnum, getBitWidth } from "../utils/read-utils.js";
+import { crc32 } from "../utils/crc32.js";
+/** Shared empty level storage for required flat columns, whose levels are implicit. */
+const EMPTY_PARQUET_LEVEL_BUFFER = new Uint8Array(0);
 /**
  * Decode data pages
  * @param buffer - input data
@@ -21,61 +24,174 @@ export async function decodeDataPages(buffer, context) {
         offset: 0,
         size: buffer.length
     };
+    const expectedLevelCount = context.numValues === undefined ? undefined : Number(context.numValues);
+    if (expectedLevelCount !== undefined &&
+        (!Number.isSafeInteger(expectedLevelCount) || expectedLevelCount < 0)) {
+        throw new Error(`Invalid Parquet column value count ${expectedLevelCount}`);
+    }
+    const outputCapacity = expectedLevelCount ?? 0;
+    const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
     const data = {
-        rlevels: [],
-        dlevels: [],
-        values: [],
+        rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
+        dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
+        values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
         pageHeaders: [],
-        count: 0
+        count: 0,
+        nullCount: 0
     };
     let dictionary = context.dictionary || [];
+    let levelOffset = 0;
+    let valueOffset = 0;
     while (
     // @ts-ignore size can be undefined
     cursor.offset < cursor.size &&
-        (!context.numValues || data.dlevels.length < Number(context.numValues))) {
+        (expectedLevelCount === undefined || levelOffset < expectedLevelCount)) {
         // Looks like we have to decode these in sequence due to cursor updates?
-        const page = await decodePage(cursor, context);
+        const target = {
+            values: data.values,
+            valueOffset,
+            byteArrayOutput,
+            dictionary,
+            rlevels: data.rlevels,
+            dlevels: data.dlevels,
+            levelOffset
+        };
+        const page = context.compression === 'UNCOMPRESSED'
+            ? decodeUncompressedPage(cursor, context, target)
+            : await decodePage(cursor, context, target);
         if (page.dictionary) {
             dictionary = page.dictionary;
             // eslint-disable-next-line no-continue
             continue;
         }
-        const valueEncoding = getThriftEnum(Encoding, page.pageHeader.data_page_header?.encoding);
-        // Pages might be in different encodings. We don't need to decode in case
-        // of 'PLAIN' encoding because all values are already in place
-        if (dictionary.length && valueEncoding !== 'PLAIN') {
-            // eslint-disable-next-line no-loop-func
-            page.values = page.values.map((value) => dictionary[value]);
+        levelOffset += page.count;
+        if (page.directValuesWritten !== undefined) {
+            valueOffset += page.directValuesWritten;
+            data.nullCount += page.count - page.directValuesWritten;
         }
-        for (let index = 0; index < page.rlevels.length; index++) {
-            data.rlevels.push(page.rlevels[index]);
-            data.dlevels.push(page.dlevels[index]);
-            const value = page.values[index];
-            if (value !== undefined) {
-                data.values.push(value);
+        else {
+            const valueEncoding = getThriftEnum(Encoding, page.pageHeader.data_page_header?.encoding ?? page.pageHeader.data_page_header_v2?.encoding);
+            const usesDictionary = dictionary.length &&
+                (valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY');
+            for (let index = 0; index < page.values.length; index++) {
+                const value = usesDictionary ? dictionary[Number(page.values[index])] : page.values[index];
+                if (value !== undefined) {
+                    data.values[valueOffset++] = value;
+                }
             }
         }
         data.count += page.count;
         data.pageHeaders.push(page.pageHeader);
     }
+    data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
+    data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
+    if (byteArrayOutput) {
+        data.values = [];
+        data.byteArrayData = {
+            data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
+            valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
+        };
+    }
+    else {
+        data.values = trimParquetValueBuffer(data.values, valueOffset);
+    }
     return data;
 }
+/**
+ * Decodes an uncompressed column chunk synchronously.
+ *
+ * In-memory Parquet reads use this entry point to avoid one resolved Promise per selected column.
+ */
+export function decodeUncompressedDataPages(buffer, context) {
+    const cursor = { buffer, offset: 0, size: buffer.length };
+    const expectedLevelCount = context.numValues === undefined ? undefined : Number(context.numValues);
+    if (expectedLevelCount !== undefined &&
+        (!Number.isSafeInteger(expectedLevelCount) || expectedLevelCount < 0)) {
+        throw new Error(`Invalid Parquet column value count ${expectedLevelCount}`);
+    }
+    const outputCapacity = expectedLevelCount ?? 0;
+    const byteArrayOutput = createArrowByteArrayOutput(buffer, context, outputCapacity);
+    const data = {
+        rlevels: createParquetLevelBuffer(context, outputCapacity, context.rLevelMax),
+        dlevels: createParquetLevelBuffer(context, outputCapacity, context.dLevelMax),
+        values: byteArrayOutput ? [] : createParquetColumnValueBuffer(context, outputCapacity),
+        pageHeaders: [],
+        count: 0,
+        nullCount: 0
+    };
+    let dictionary = context.dictionary || [];
+    let levelOffset = 0;
+    let valueOffset = 0;
+    while (cursor.offset < (cursor.size ?? cursor.buffer.length) &&
+        (expectedLevelCount === undefined || levelOffset < expectedLevelCount)) {
+        const page = decodeUncompressedPage(cursor, context, {
+            values: data.values,
+            valueOffset,
+            byteArrayOutput,
+            dictionary,
+            rlevels: data.rlevels,
+            dlevels: data.dlevels,
+            levelOffset
+        });
+        if (page.dictionary) {
+            dictionary = page.dictionary;
+            continue;
+        }
+        levelOffset += page.count;
+        if (page.directValuesWritten !== undefined) {
+            valueOffset += page.directValuesWritten;
+            data.nullCount += page.count - page.directValuesWritten;
+        }
+        else {
+            const valueEncoding = getThriftEnum(Encoding, page.pageHeader.data_page_header?.encoding ?? page.pageHeader.data_page_header_v2?.encoding);
+            const usesDictionary = dictionary.length &&
+                (valueEncoding === 'PLAIN_DICTIONARY' || valueEncoding === 'RLE_DICTIONARY');
+            for (let index = 0; index < page.values.length; index++) {
+                const value = usesDictionary ? dictionary[Number(page.values[index])] : page.values[index];
+                if (value !== undefined)
+                    data.values[valueOffset++] = value;
+            }
+        }
+        data.count += page.count;
+        data.pageHeaders.push(page.pageHeader);
+    }
+    data.rlevels = trimParquetLevelBuffer(data.rlevels, levelOffset);
+    data.dlevels = trimParquetLevelBuffer(data.dlevels, levelOffset);
+    if (byteArrayOutput) {
+        data.values = [];
+        data.byteArrayData = {
+            data: byteArrayOutput.data.subarray(0, byteArrayOutput.byteLength),
+            valueOffsets: byteArrayOutput.valueOffsets.subarray(0, valueOffset + 1)
+        };
+    }
+    else {
+        data.values = trimParquetValueBuffer(data.values, valueOffset);
+    }
+    return data;
+}
+/** Decodes a dictionary page from an uncompressed in-memory column chunk. */
+export function decodeUncompressedDictionaryBuffer(buffer, context) {
+    const cursor = { buffer, offset: 0, size: buffer.length };
+    return decodeUncompressedPage(cursor, context).dictionary;
+}
 /**
  * Decode parquet page based on page type
  * @param cursor
  * @param context
  */
-export async function decodePage(cursor, context) {
+export async function decodePage(cursor, context, target) {
     let page;
     const { pageHeader, length } = decodePageHeader(cursor.buffer, cursor.offset);
     cursor.offset += length;
     const pageType = getThriftEnum(PageType, pageHeader.type);
+    const pageEnd = cursor.offset + pageHeader.compressed_page_size;
+    verifyPageChecksum(cursor, pageHeader, pageEnd, context);
     switch (pageType) {
         case 'DATA_PAGE':
-            page = await decodeDataPage(cursor, pageHeader, context);
+            page = await decodeDataPage(cursor, pageHeader, context, target);
             break;
         case 'DATA_PAGE_V2':
-            page = await decodeDataPageV2(cursor, pageHeader, context);
+            page = await decodeDataPageV2(cursor, pageHeader, context, target);
             break;
         case 'DICTIONARY_PAGE':
             page = {
@@ -88,6 +204,46 @@ export async function decodePage(cursor, context) {
     }
     return page;
 }
+/** Decodes one uncompressed page without introducing a Promise/microtask boundary. */
+function decodeUncompressedPage(cursor, context, target) {
+    const { pageHeader, length } = decodePageHeader(cursor.buffer, cursor.offset);
+    cursor.offset += length;
+    const pageEnd = cursor.offset + pageHeader.compressed_page_size;
+    verifyPageChecksum(cursor, pageHeader, pageEnd, context);
+    const pageType = getThriftEnum(PageType, pageHeader.type);
+    switch (pageType) {
+        case 'DATA_PAGE':
+            return decodeDataPageValues(cursor, pageHeader, context, target);
+        case 'DATA_PAGE_V2':
+            return decodeUncompressedDataPageV2(cursor, pageHeader, context, target);
+        case 'DICTIONARY_PAGE':
+            return {
+                dictionary: decodeUncompressedDictionaryPage(cursor, pageHeader, context),
+                dlevels: EMPTY_PARQUET_LEVEL_BUFFER,
+                rlevels: EMPTY_PARQUET_LEVEL_BUFFER,
+                values: [],
+                count: 0,
+                pageHeader
+            };
+        default:
+            throw new Error(`invalid page type: ${pageType}`);
+    }
+}
+/** Verifies a page checksum before either the synchronous or asynchronous decode path consumes it. */
+function verifyPageChecksum(cursor, pageHeader, pageEnd, context) {
+    if (!context.verifyPageChecksums)
+        return;
+    if (pageEnd > (cursor.size ?? cursor.buffer.length)) {
+        throw new Error('Parquet page extends beyond the available buffer');
+    }
+    if (pageHeader.crc === undefined)
+        return;
+    const expected = pageHeader.crc >>> 0;
+    const actual = crc32(cursor.buffer.subarray(cursor.offset, pageEnd));
+    if (actual !== expected) {
+        throw new Error(`Parquet page checksum mismatch: expected ${expected}, calculated ${actual}`);
+    }
+}
 /**
  * Decode parquet schema
  * @param schemaElements input schema elements data
@@ -122,29 +278,28 @@ export function decodeSchema(schemaElements, offset, len) {
             const res = decodeSchema(schemaElements, next + 1, schemaElement.num_children);
             next = res.next;
             schema[schemaElement.name] = {
-                // type: undefined,
                 optional,
                 repeated,
+                logicalType: decodeSchemaLogicalType(schemaElement),
+                fieldId: schemaElement.field_id,
                 fields: res.schema
             };
         }
         else {
-            const type = getThriftEnum(Type, schemaElement.type);
-            let logicalType = type;
-            if (schemaElement.converted_type) {
-                logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);
-            }
-            switch (logicalType) {
-                case 'DECIMAL':
-                    logicalType = `${logicalType}_${type}`;
-                    break;
-                default:
-            }
+            const physicalType = getThriftEnum(Type, schemaElement.type);
+            const logicalType = decodeSchemaLogicalType(schemaElement);
+            const parquetType = getSchemaParquetType(schemaElement, physicalType, logicalType);
+            const precision = logicalType?.precision ?? schemaElement.precision;
+            const scale = logicalType?.scale ?? schemaElement.scale;
             schema[schemaElement.name] = {
-                type: logicalType,
+                type: parquetType,
+                physicalType,
                 typeLength: schemaElement.type_length,
-                presision: schemaElement.precision,
-                scale: schemaElement.scale,
+                presision: precision,
+                precision,
+                scale,
+                logicalType,
+                fieldId: schemaElement.field_id,
                 optional,
                 repeated
             };
@@ -153,6 +308,171 @@ export function decodeSchema(schemaElements, offset, len) {
     }
     return { schema, offset, next };
 }
+/** Decodes the parameterized Parquet LogicalType union, falling back to ConvertedType metadata. */
+function decodeSchemaLogicalType(schemaElement) {
+    const logicalType = schemaElement.logicalType;
+    if (logicalType?.STRING)
+        return { type: 'STRING' };
+    if (logicalType?.MAP)
+        return { type: 'MAP' };
+    if (logicalType?.LIST)
+        return { type: 'LIST' };
+    if (logicalType?.ENUM)
+        return { type: 'ENUM' };
+    if (logicalType?.DECIMAL) {
+        return {
+            type: 'DECIMAL',
+            precision: logicalType.DECIMAL.precision,
+            scale: logicalType.DECIMAL.scale
+        };
+    }
+    if (logicalType?.DATE)
+        return { type: 'DATE' };
+    if (logicalType?.TIME) {
+        return {
+            type: 'TIME',
+            unit: decodeTimeUnit(logicalType.TIME.unit),
+            isAdjustedToUTC: logicalType.TIME.isAdjustedToUTC
+        };
+    }
+    if (logicalType?.TIMESTAMP) {
+        return {
+            type: 'TIMESTAMP',
+            unit: decodeTimeUnit(logicalType.TIMESTAMP.unit),
+            isAdjustedToUTC: logicalType.TIMESTAMP.isAdjustedToUTC
+        };
+    }
+    if (logicalType?.INTEGER) {
+        const bitWidth = logicalType.INTEGER.bitWidth;
+        if (bitWidth !== 8 && bitWidth !== 16 && bitWidth !== 32 && bitWidth !== 64) {
+            throw new Error(`parquet: invalid INTEGER bit width ${bitWidth}`);
+        }
+        return { type: 'INTEGER', bitWidth, isSigned: logicalType.INTEGER.isSigned };
+    }
+    if (logicalType?.UNKNOWN)
+        return { type: 'UNKNOWN' };
+    if (logicalType?.JSON)
+        return { type: 'JSON' };
+    if (logicalType?.BSON)
+        return { type: 'BSON' };
+    if (logicalType?.UUID)
+        return { type: 'UUID' };
+    if (logicalType?.FLOAT16)
+        return { type: 'FLOAT16' };
+    if (logicalType?.VARIANT) {
+        return { type: 'VARIANT', specificationVersion: logicalType.VARIANT.specification_version };
+    }
+    if (logicalType?.GEOMETRY)
+        return { type: 'GEOMETRY', crs: logicalType.GEOMETRY.crs };
+    if (logicalType?.GEOGRAPHY) {
+        return {
+            type: 'GEOGRAPHY',
+            crs: logicalType.GEOGRAPHY.crs,
+            algorithm: logicalType.GEOGRAPHY.algorithm === undefined
+                ? undefined
+                : getThriftEnum(EdgeInterpolationAlgorithm, logicalType.GEOGRAPHY.algorithm)
+        };
+    }
+    if (schemaElement.converted_type === undefined || schemaElement.converted_type === null) {
+        return undefined;
+    }
+    const convertedType = getThriftEnum(ConvertedType, schemaElement.converted_type);
+    switch (convertedType) {
+        case 'UTF8':
+            return { type: 'STRING' };
+        case 'MAP':
+        case 'MAP_KEY_VALUE':
+            return { type: 'MAP' };
+        case 'LIST':
+            return { type: 'LIST' };
+        case 'ENUM':
+            return { type: 'ENUM' };
+        case 'DECIMAL':
+            return { type: 'DECIMAL', precision: schemaElement.precision, scale: schemaElement.scale };
+        case 'DATE':
+            return { type: 'DATE' };
+        case 'TIME_MILLIS':
+            return { type: 'TIME', unit: 'MILLIS', isAdjustedToUTC: true };
+        case 'TIME_MICROS':
+            return { type: 'TIME', unit: 'MICROS', isAdjustedToUTC: true };
+        case 'TIMESTAMP_MILLIS':
+            return { type: 'TIMESTAMP', unit: 'MILLIS', isAdjustedToUTC: true };
+        case 'TIMESTAMP_MICROS':
+            return { type: 'TIMESTAMP', unit: 'MICROS', isAdjustedToUTC: true };
+        case 'UINT_8':
+        case 'UINT_16':
+        case 'UINT_32':
+        case 'UINT_64':
+            return {
+                type: 'INTEGER',
+                bitWidth: Number(convertedType.slice(5)),
+                isSigned: false
+            };
+        case 'INT_8':
+        case 'INT_16':
+        case 'INT_32':
+        case 'INT_64':
+            return {
+                type: 'INTEGER',
+                bitWidth: Number(convertedType.slice(4)),
+                isSigned: true
+            };
+        case 'JSON':
+            return { type: 'JSON' };
+        case 'BSON':
+            return { type: 'BSON' };
+        default:
+            return undefined;
+    }
+}
+/** Resolves one physical/logical pair to the internal Parquet value converter. */
+function getSchemaParquetType(schemaElement, physicalType, logicalType) {
+    if (!logicalType) {
+        return schemaElement.converted_type === undefined || schemaElement.converted_type === null
+            ? physicalType
+            : getThriftEnum(ConvertedType, schemaElement.converted_type);
+    }
+    switch (logicalType.type) {
+        case 'STRING':
+            return 'UTF8';
+        case 'ENUM':
+            return 'ENUM';
+        case 'DECIMAL':
+            return `DECIMAL_${physicalType}`;
+        case 'DATE':
+            return 'DATE';
+        case 'TIME':
+            return `TIME_${logicalType.unit}`;
+        case 'TIMESTAMP':
+            return `TIMESTAMP_${logicalType.unit}`;
+        case 'INTEGER':
+            return `${logicalType.isSigned ? 'INT' : 'UINT'}_${logicalType.bitWidth}`;
+        case 'UNKNOWN':
+        case 'JSON':
+        case 'BSON':
+        case 'UUID':
+        case 'FLOAT16':
+        case 'VARIANT':
+        case 'GEOMETRY':
+        case 'GEOGRAPHY':
+            return logicalType.type;
+        default:
+            if (schemaElement.converted_type !== undefined && schemaElement.converted_type !== null) {
+                return getThriftEnum(ConvertedType, schemaElement.converted_type);
+            }
+            return physicalType;
+    }
+}
+/** Returns the selected member of the Parquet TimeUnit union. */
+function decodeTimeUnit(timeUnit) {
+    if (timeUnit?.MILLIS)
+        return 'MILLIS';
+    if (timeUnit?.MICROS)
+        return 'MICROS';
+    if (timeUnit?.NANOS)
+        return 'NANOS';
+    throw new Error('parquet: TIME or TIMESTAMP logical type is missing its unit');
+}
 /**
  * Decode a consecutive array of data using one of the parquet encodings
  */
@@ -168,13 +488,15 @@ function decodeValues(type, encoding, cursor, count, opts) {
  * @param header
  * @param options
  */
-async function decodeDataPage(cursor, header, context) {
+async function decodeDataPage(cursor, header, context, target) {
     const cursorEnd = cursor.offset + header.compressed_page_size;
-    const valueCount = header.data_page_header?.num_values;
     /* uncompress page */
     let dataCursor = cursor;
     if (context.compression !== 'UNCOMPRESSED') {
-        const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
+        const compressedValues = cursor.buffer.slice(cursor.offset, cursorEnd);
+        const valuesBuf = context.decompressPage
+            ? await context.decompressPage(compressedValues, header.uncompressed_page_size)
+            : await decompress(context.compression, compressedValues, header.uncompressed_page_size);
         dataCursor = {
             buffer: valuesBuf,
             offset: 0,
@@ -182,51 +504,47 @@ async function decodeDataPage(cursor, header, context) {
         };
         cursor.offset = cursorEnd;
     }
+    return decodeDataPageValues(dataCursor, header, context, target);
+}
+/** Decodes one uncompressed Data Page V1 body without introducing an asynchronous boundary. */
+function decodeDataPageValues(dataCursor, header, context, target) {
+    const valueCount = header.data_page_header?.num_values;
     /* read repetition levels */
     const rLevelEncoding = getThriftEnum(Encoding, header.data_page_header?.repetition_level_encoding);
-    // tslint:disable-next-line:prefer-array-literal
-    let rLevels = new Array(valueCount);
-    if (context.column.rLevelMax > 0) {
-        rLevels = decodeValues(PARQUET_RDLVL_TYPE, rLevelEncoding, dataCursor, valueCount, {
-            bitWidth: getBitWidth(context.column.rLevelMax),
-            disableEnvelope: false
-            // column: opts.column
-        });
-    }
-    else {
-        rLevels.fill(0);
-    }
+    const rLevels = decodeLevels(dataCursor, valueCount, context.column.rLevelMax, rLevelEncoding, false, target?.rlevels, target?.levelOffset);
     /* read definition levels */
     const dLevelEncoding = getThriftEnum(Encoding, header.data_page_header?.definition_level_encoding);
-    // tslint:disable-next-line:prefer-array-literal
-    let dLevels = new Array(valueCount);
+    const dLevels = decodeLevels(dataCursor, valueCount, context.column.dLevelMax, dLevelEncoding, false, target?.dlevels, target?.levelOffset);
+    let valueCountNonNull = valueCount;
     if (context.column.dLevelMax > 0) {
-        dLevels = decodeValues(PARQUET_RDLVL_TYPE, dLevelEncoding, dataCursor, valueCount, {
-            bitWidth: getBitWidth(context.column.dLevelMax),
-            disableEnvelope: false
-            // column: opts.column
-        });
-    }
-    else {
-        dLevels.fill(0);
-    }
-    let valueCountNonNull = 0;
-    for (const dlvl of dLevels) {
-        if (dlvl === context.column.dLevelMax) {
-            valueCountNonNull++;
+        valueCountNonNull = 0;
+        const decodedDefinitionLevels = target?.dlevels || dLevels;
+        const definitionLevelOffset = target?.levelOffset || 0;
+        for (let index = 0; index < valueCount; index++) {
+            if (decodedDefinitionLevels[definitionLevelOffset + index] === context.column.dLevelMax) {
+                valueCountNonNull++;
+            }
         }
     }
     /* read values */
     const valueEncoding = getThriftEnum(Encoding, header.data_page_header?.encoding);
     const decodeOptions = {
         typeLength: context.column.typeLength,
-        bitWidth: context.column.typeLength
+        bitWidth: getValueBitWidth(context, valueEncoding),
+        retainByteArrayViews: context.retainByteArrayViews,
+        output: target?.values,
+        outputOffset: target?.valueOffset,
+        dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
+        byteArrayOutput: target?.byteArrayOutput,
+        int64AsBigInt: shouldDecodeInt64AsBigInt(context),
+        int96AsTimestamp: context.int96AsTimestamp
     };
     const values = decodeValues(context.column.primitiveType, valueEncoding, dataCursor, valueCountNonNull, decodeOptions);
     return {
         dlevels: dLevels,
         rlevels: rLevels,
         values,
+        directValuesWritten: target ? valueCountNonNull : undefined,
         count: valueCount,
         pageHeader: header
     };
@@ -238,60 +556,298 @@ async function decodeDataPage(cursor, header, context) {
  * @param opts
  * @returns
  */
-async function decodeDataPageV2(cursor, header, context) {
+async function decodeDataPageV2(cursor, header, context, target) {
+    const dataPageHeader = header.data_page_header_v2;
+    if (!dataPageHeader) {
+        throw new Error('Missing Parquet data page v2 header');
+    }
     const cursorEnd = cursor.offset + header.compressed_page_size;
-    const valueCount = header.data_page_header_v2?.num_values;
-    // @ts-ignore
-    const valueCountNonNull = valueCount - header.data_page_header_v2?.num_nulls;
-    const valueEncoding = getThriftEnum(Encoding, header.data_page_header_v2?.encoding);
+    const levelsOffset = cursor.offset;
+    const valueCount = dataPageHeader.num_values;
+    const valueCountNonNull = valueCount - dataPageHeader.num_nulls;
+    const valueEncoding = getThriftEnum(Encoding, dataPageHeader.encoding);
+    if (header.compressed_page_size < 0 ||
+        cursorEnd > (cursor.size ?? cursor.buffer.length) ||
+        valueCountNonNull < 0) {
+        throw new Error('Invalid Parquet data page v2 header');
+    }
     /* read repetition levels */
-    // tslint:disable-next-line:prefer-array-literal
-    let rLevels = new Array(valueCount);
+    let rLevels = [];
     if (context.column.rLevelMax > 0) {
-        rLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
-            bitWidth: getBitWidth(context.column.rLevelMax),
-            disableEnvelope: true
-        });
+        const repetitionLevelCursor = createPageSliceCursor(cursor, levelsOffset, dataPageHeader.repetition_levels_byte_length);
+        rLevels = decodeLevels(repetitionLevelCursor, valueCount, context.column.rLevelMax, PARQUET_RDLVL_ENCODING, true, target?.rlevels, target?.levelOffset);
     }
-    else {
+    else if (!target) {
+        rLevels = new Array(valueCount);
         rLevels.fill(0);
     }
+    const definitionLevelsOffset = levelsOffset + dataPageHeader.repetition_levels_byte_length;
     /* read definition levels */
-    // tslint:disable-next-line:prefer-array-literal
-    let dLevels = new Array(valueCount);
+    let dLevels = [];
     if (context.column.dLevelMax > 0) {
-        dLevels = decodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, cursor, valueCount, {
-            bitWidth: getBitWidth(context.column.dLevelMax),
-            disableEnvelope: true
-        });
+        const definitionLevelCursor = createPageSliceCursor(cursor, definitionLevelsOffset, dataPageHeader.definition_levels_byte_length);
+        dLevels = decodeLevels(definitionLevelCursor, valueCount, context.column.dLevelMax, PARQUET_RDLVL_ENCODING, true, target?.dlevels, target?.levelOffset);
     }
-    else {
+    else if (!target) {
+        dLevels = new Array(valueCount);
         dLevels.fill(0);
     }
     /* read values */
-    let valuesBufCursor = cursor;
-    if (header.data_page_header_v2?.is_compressed) {
-        const valuesBuf = await decompress(context.compression, cursor.buffer.slice(cursor.offset, cursorEnd), header.uncompressed_page_size);
-        valuesBufCursor = {
-            buffer: valuesBuf,
-            offset: 0,
-            size: valuesBuf.length
-        };
-        cursor.offset = cursorEnd;
+    const valuesOffset = definitionLevelsOffset + dataPageHeader.definition_levels_byte_length;
+    const valuesCompressedByteLength = cursorEnd - valuesOffset;
+    const valuesUncompressedByteLength = header.uncompressed_page_size -
+        dataPageHeader.repetition_levels_byte_length -
+        dataPageHeader.definition_levels_byte_length;
+    if (valuesCompressedByteLength < 0 || valuesUncompressedByteLength < 0) {
+        throw new Error('Invalid Parquet data page v2 level lengths');
+    }
+    let valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
+    if (dataPageHeader.is_compressed !== false && context.compression !== 'UNCOMPRESSED') {
+        valuesBuffer = context.decompressPage
+            ? await context.decompressPage(valuesBuffer, valuesUncompressedByteLength)
+            : await decompress(context.compression, valuesBuffer, valuesUncompressedByteLength);
     }
+    cursor.offset = cursorEnd;
+    return decodeDataPageV2Values(valuesBuffer, header, context, target, rLevels, dLevels, valueCount, valueCountNonNull, valueEncoding);
+}
+/** Decodes one uncompressed Data Page V2 without awaiting an unused decompressor. */
+function decodeUncompressedDataPageV2(cursor, header, context, target) {
+    const dataPageHeader = header.data_page_header_v2;
+    if (!dataPageHeader) {
+        throw new Error('Missing Parquet data page v2 header');
+    }
+    const cursorEnd = cursor.offset + header.compressed_page_size;
+    const levelsOffset = cursor.offset;
+    const valueCount = dataPageHeader.num_values;
+    const valueCountNonNull = valueCount - dataPageHeader.num_nulls;
+    const valueEncoding = getThriftEnum(Encoding, dataPageHeader.encoding);
+    if (header.compressed_page_size < 0 ||
+        cursorEnd > (cursor.size ?? cursor.buffer.length) ||
+        valueCountNonNull < 0) {
+        throw new Error('Invalid Parquet data page v2 header');
+    }
+    let rLevels = [];
+    if (context.column.rLevelMax > 0) {
+        const repetitionLevelCursor = createPageSliceCursor(cursor, levelsOffset, dataPageHeader.repetition_levels_byte_length);
+        rLevels = decodeLevels(repetitionLevelCursor, valueCount, context.column.rLevelMax, PARQUET_RDLVL_ENCODING, true, target?.rlevels, target?.levelOffset);
+    }
+    else if (!target) {
+        rLevels = new Array(valueCount).fill(0);
+    }
+    const definitionLevelsOffset = levelsOffset + dataPageHeader.repetition_levels_byte_length;
+    let dLevels = [];
+    if (context.column.dLevelMax > 0) {
+        const definitionLevelCursor = createPageSliceCursor(cursor, definitionLevelsOffset, dataPageHeader.definition_levels_byte_length);
+        dLevels = decodeLevels(definitionLevelCursor, valueCount, context.column.dLevelMax, PARQUET_RDLVL_ENCODING, true, target?.dlevels, target?.levelOffset);
+    }
+    else if (!target) {
+        dLevels = new Array(valueCount).fill(0);
+    }
+    const valuesOffset = definitionLevelsOffset + dataPageHeader.definition_levels_byte_length;
+    const valuesUncompressedByteLength = header.uncompressed_page_size -
+        dataPageHeader.repetition_levels_byte_length -
+        dataPageHeader.definition_levels_byte_length;
+    if (valuesOffset > cursorEnd || valuesUncompressedByteLength < 0) {
+        throw new Error('Invalid Parquet data page v2 level lengths');
+    }
+    const valuesBuffer = cursor.buffer.subarray(valuesOffset, cursorEnd);
+    cursor.offset = cursorEnd;
+    return decodeDataPageV2Values(valuesBuffer, header, context, target, rLevels, dLevels, valueCount, valueCountNonNull, valueEncoding);
+}
+/** Decodes a prepared Data Page V2 value section without an asynchronous boundary. */
+function decodeDataPageV2Values(valuesBuffer, header, context, target, rLevels, dLevels, valueCount, valueCountNonNull, valueEncoding) {
+    const valuesBufCursor = { buffer: valuesBuffer, offset: 0, size: valuesBuffer.length };
     const decodeOptions = {
         typeLength: context.column.typeLength,
-        bitWidth: context.column.typeLength
+        bitWidth: getValueBitWidth(context, valueEncoding),
+        retainByteArrayViews: context.retainByteArrayViews,
+        output: target?.values,
+        outputOffset: target?.valueOffset,
+        dictionary: isDictionaryEncoding(valueEncoding) ? target?.dictionary : undefined,
+        byteArrayOutput: target?.byteArrayOutput,
+        int64AsBigInt: shouldDecodeInt64AsBigInt(context),
+        int96AsTimestamp: context.int96AsTimestamp
     };
     const values = decodeValues(context.column.primitiveType, valueEncoding, valuesBufCursor, valueCountNonNull, decodeOptions);
     return {
         dlevels: dLevels,
         rlevels: rLevels,
         values,
+        directValuesWritten: target ? valueCountNonNull : undefined,
         count: valueCount,
         pageHeader: header
     };
 }
+/** Decodes repetition or definition levels into an optional column-level destination. */
+function decodeLevels(cursor, count, levelMax, encoding, disableEnvelope, output, outputOffset = 0) {
+    if (levelMax === 0) {
+        return output ? [] : new Array(count).fill(0);
+    }
+    const levels = decodeValues(PARQUET_RDLVL_TYPE, encoding, cursor, count, {
+        bitWidth: getBitWidth(levelMax),
+        disableEnvelope,
+        output,
+        outputOffset
+    });
+    return output ? [] : levels;
+}
+/**
+ * Allocates a compact Arrow-compatible destination for supported uncompressed byte pages.
+ *
+ * Scanning only the small Thrift page headers avoids committing to the fast path when a column
+ * mixes in an encoding that still requires materialized values. The page buffer is a useful
+ * initial capacity; the destination grows for prefix-reconstructed delta values when necessary.
+ */
+function createArrowByteArrayOutput(buffer, context, valueCapacity) {
+    if (!context.useArrowByteArrayBuffers ||
+        context.compression !== 'UNCOMPRESSED' ||
+        !supportsArrowByteArrayOutput(context.column) ||
+        valueCapacity === 0) {
+        return undefined;
+    }
+    if (!context.hasOnlyArrowByteArrayDataPages &&
+        !hasOnlyArrowByteArrayDataPages(buffer, valueCapacity)) {
+        return undefined;
+    }
+    return {
+        data: context.column.primitiveType === 'FIXED_LEN_BYTE_ARRAY'
+            ? new Uint8Array(0)
+            : new Uint8Array(Math.max(1, buffer.byteLength)),
+        valueOffsets: new Int32Array(valueCapacity + 1),
+        byteLength: 0
+    };
+}
+/** Returns whether a BYTE_ARRAY field maps directly to Arrow Utf8 or Binary without conversion. */
+function supportsArrowByteArrayOutput(column) {
+    if (column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY') {
+        return false;
+    }
+    if (column.logicalType) {
+        return (column.logicalType.type === 'STRING' ||
+            column.logicalType.type === 'ENUM' ||
+            column.logicalType.type === 'JSON' ||
+            column.logicalType.type === 'BSON' ||
+            column.logicalType.type === 'VARIANT' ||
+            column.logicalType.type === 'GEOMETRY' ||
+            column.logicalType.type === 'GEOGRAPHY');
+    }
+    return (!column.originalType ||
+        column.originalType === 'UTF8' ||
+        column.originalType === 'ENUM' ||
+        column.originalType === 'JSON' ||
+        column.originalType === 'BSON' ||
+        column.originalType === 'INTERVAL');
+}
+/** Returns whether a complete column buffer contains only direct Arrow byte-array data pages. */
+function hasOnlyArrowByteArrayDataPages(buffer, expectedLevelCount) {
+    let offset = 0;
+    let levelCount = 0;
+    while (offset < buffer.byteLength && levelCount < expectedLevelCount) {
+        const { pageHeader, length } = decodePageHeader(buffer, offset);
+        const pageType = getThriftEnum(PageType, pageHeader.type);
+        if (pageType !== 'DATA_PAGE' && pageType !== 'DATA_PAGE_V2') {
+            return false;
+        }
+        const encoding = getThriftEnum(Encoding, pageHeader.data_page_header?.encoding ?? pageHeader.data_page_header_v2?.encoding);
+        if (encoding !== 'PLAIN' &&
+            encoding !== 'DELTA_LENGTH_BYTE_ARRAY' &&
+            encoding !== 'DELTA_BYTE_ARRAY') {
+            return false;
+        }
+        const valueCount = pageHeader.data_page_header?.num_values ?? pageHeader.data_page_header_v2?.num_values;
+        if (valueCount === undefined || valueCount < 0) {
+            return false;
+        }
+        levelCount += valueCount;
+        offset += length + pageHeader.compressed_page_size;
+        if (offset > buffer.byteLength) {
+            return false;
+        }
+    }
+    return levelCount === expectedLevelCount;
+}
+/** Allocates compact unsigned storage for repetition and definition levels. */
+function createParquetLevelBuffer(context, capacity, levelMax) {
+    if (context.useTypedLevelBuffers && levelMax === 0) {
+        // Required flat columns have no encoded levels. Avoid allocating two row-sized zero buffers
+        // that neither direct Arrow construction nor level decoding will inspect.
+        return EMPTY_PARQUET_LEVEL_BUFFER;
+    }
+    if (!context.useTypedLevelBuffers || capacity === 0) {
+        return levelMax > 0 ? new Array(capacity) : new Array(capacity).fill(0);
+    }
+    if (levelMax <= 0xff) {
+        return new Uint8Array(capacity);
+    }
+    if (levelMax <= 0xffff) {
+        return new Uint16Array(capacity);
+    }
+    return new Uint32Array(capacity);
+}
+/** Restricts an overallocated level buffer to the number of decoded levels. */
+function trimParquetLevelBuffer(levels, length) {
+    if (Array.isArray(levels)) {
+        levels.length = length;
+        return levels;
+    }
+    if (levels.length === 0 || levels.length === length)
+        return levels;
+    return levels.subarray(0, length);
+}
+/** Returns whether an encoding stores RLE dictionary indices instead of physical values. */
+function isDictionaryEncoding(encoding) {
+    return encoding === 'PLAIN_DICTIONARY' || encoding === 'RLE_DICTIONARY';
+}
+/** Preserves exact physical signed INT64 values in every output shape. */
+function shouldDecodeInt64AsBigInt(context) {
+    return context.column.primitiveType === 'INT64';
+}
+/** Allocates the narrowest lossless column buffer supported by the current JavaScript decoder. */
+function createParquetColumnValueBuffer(context, capacity) {
+    if (!context.useTypedValueBuffers || capacity === 0) {
+        return new Array(capacity);
+    }
+    switch (context.column.primitiveType) {
+        case 'BOOLEAN':
+            return new Uint8Array(capacity);
+        case 'INT32':
+            return new Int32Array(capacity);
+        case 'INT64':
+            return new BigInt64Array(capacity);
+        case 'INT96':
+            return context.int96AsTimestamp ? new BigInt64Array(capacity) : new Float64Array(capacity);
+        case 'DOUBLE':
+            return new Float64Array(capacity);
+        case 'FLOAT':
+            return new Float32Array(capacity);
+        default:
+            return new Array(capacity);
+    }
+}
+/** Restricts an overallocated typed column buffer to its decoded non-null values. */
+function trimParquetValueBuffer(values, length) {
+    if (Array.isArray(values)) {
+        values.length = length;
+        return values;
+    }
+    return values.subarray(0, length);
+}
+/** Returns the physical value bit width required by encodings such as RLE BOOLEAN. */
+function getValueBitWidth(context, encoding) {
+    if (encoding === 'RLE' && context.column.primitiveType === 'BOOLEAN') {
+        return 1;
+    }
+    return context.column.typeLength;
+}
+/** Creates a bounded cursor over one uncompressed data page v2 level section. */
+function createPageSliceCursor(cursor, offset, byteLength) {
+    const end = offset + byteLength;
+    if (byteLength < 0 || end > (cursor.size ?? cursor.buffer.length)) {
+        throw new Error('Invalid Parquet data page v2 level lengths');
+    }
+    const buffer = cursor.buffer.subarray(offset, end);
+    return { buffer, offset: 0, size: buffer.length };
+}
 /**
  * Do decoding of dictionary page which helps to iterate over all indexes and get dataPage values.
  * @param cursor
@@ -307,7 +863,10 @@ async function decodeDictionaryPage(cursor, pageHeader, context) {
     };
     cursor.offset = cursorEnd;
     if (context.compression !== 'UNCOMPRESSED') {
-        const valuesBuf = await decompress(context.compression, dictCursor.buffer.slice(dictCursor.offset, cursorEnd), pageHeader.uncompressed_page_size);
+        const compressedValues = dictCursor.buffer.subarray(dictCursor.offset);
+        const valuesBuf = context.decompressPage
+            ? await context.decompressPage(compressedValues, pageHeader.uncompressed_page_size)
+            : await decompress(context.compression, compressedValues, pageHeader.uncompressed_page_size);
         dictCursor = {
             buffer: valuesBuf,
             offset: 0,
@@ -315,28 +874,37 @@ async function decodeDictionaryPage(cursor, pageHeader, context) {
         };
         cursor.offset = cursorEnd;
     }
-    const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
-    const decodedDictionaryValues = decodeValues(context.column.primitiveType, context.column.encoding, dictCursor, numValues, 
-    // TODO - this looks wrong?
-    context);
-    // Makes it look a little easier
-    let values;
-    if (context?.preserveBinary) {
-        values = decodedDictionaryValues.map((d) => preserveBinary(d));
-    }
-    else {
-        values = decodedDictionaryValues.map((d) => d.toString());
-    }
-    return values;
+    return decodeDictionaryValues(dictCursor, pageHeader, context);
 }
-function preserveBinary(d) {
-    if (ArrayBuffer.isView(d)) {
-        return d;
-    }
-    // Convert to ArrayBuffer
-    if (Buffer.isBuffer(d)) {
-        return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
+/** Decodes an uncompressed dictionary page directly from a view of the column chunk. */
+function decodeUncompressedDictionaryPage(cursor, pageHeader, context) {
+    const cursorEnd = cursor.offset + pageHeader.compressed_page_size;
+    if (cursorEnd > (cursor.size ?? cursor.buffer.length)) {
+        throw new Error('Parquet dictionary page exceeds the available buffer');
     }
-    return d.toString();
+    const dictionaryCursor = {
+        buffer: cursor.buffer.subarray(cursor.offset, cursorEnd),
+        offset: 0,
+        size: pageHeader.compressed_page_size
+    };
+    cursor.offset = cursorEnd;
+    return decodeDictionaryValues(dictionaryCursor, pageHeader, context);
+}
+/** Decodes one dictionary body from its current cursor without copying uncompressed page bytes. */
+function decodeDictionaryValues(dictionaryCursor, pageHeader, context) {
+    const numValues = pageHeader?.dictionary_page_header?.num_values || 0;
+    const declaredEncoding = getThriftEnum(Encoding, pageHeader.dictionary_page_header?.encoding);
+    // Some established writers put the data-page dictionary encoding in this field even though
+    // dictionary values themselves use PLAIN. Preserve compatibility with those files.
+    const dictionaryValueEncoding = declaredEncoding === 'PLAIN_DICTIONARY' || declaredEncoding === 'RLE_DICTIONARY'
+        ? 'PLAIN'
+        : declaredEncoding;
+    const decodedDictionaryValues = decodeValues(context.column.primitiveType, dictionaryValueEncoding, dictionaryCursor, numValues, {
+        ...context,
+        typeLength: context.column.typeLength,
+        int64AsBigInt: shouldDecodeInt64AsBigInt(context),
+        int96AsTimestamp: context.int96AsTimestamp
+    });
+    return decodedDictionaryValues;
 }
 //# sourceMappingURL=decoders.js.map