@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/dist/parquetjs/encoder/parquet-encoder.js

@@ -3,21 +3,31 @@
 // Copyright (c) vis.gl contributors
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
+import { getWKBGeometryStatistics } from '@loaders.gl/gis';
 import { PARQUET_CODECS } from "../codecs/index.js";
 import * as Compression from "../compression.js";
 import * as Shred from "../schema/shred.js";
-import { BsonType, ColumnChunk, ColumnMetaData, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, DateType, DecimalType, DictionaryPageHeader, EdgeInterpolationAlgorithm, Encoding, EnumType, FieldRepetitionType, FileMetaData, Float16Type, GeographyType, GeometryType, IntType, JsonType, KeyValue, ListType, LogicalType, MapType, MicroSeconds, MilliSeconds, NanoSeconds, NullType, PageHeader, PageEncodingStats, PageType, RowGroup, SchemaElement, StringType, TimeType, TimeUnit, TimestampType, Type, UUIDType, VariantType } from "../parquet-thrift/index.js";
+import { BsonType, BoundingBox, ColumnChunk, ColumnCryptoMetaData, ColumnIndex, ColumnMetaData, SizeStatistics, Statistics, SortingColumn, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, DateType, DecimalType, DictionaryPageHeader, EdgeInterpolationAlgorithm, Encoding, EnumType, FieldRepetitionType, FileMetaData, Float16Type, GeographyType, GeospatialStatistics, GeometryType, IntType, JsonType, KeyValue, ListType, LogicalType, MapType, MicroSeconds, MilliSeconds, NanoSeconds, NullType, PageHeader, PageEncodingStats, PageType, PageLocation, RowGroup, SchemaElement, StringType, TimeType, TimeUnit, TimestampType, Type, UUIDType, VariantType, OffsetIndex } from "../parquet-thrift/index.js";
 import { osopen, oswrite, osclose } from "../utils/file-utils.js";
 import { getBitWidth, serializeThrift } from "../utils/read-utils.js";
 import { concatUint8Arrays, encodeUtf8, writeUInt32LE } from "../utils/binary-utils.js";
+import { crc32 } from "../utils/crc32.js";
 import { CompactInt64 } from "../utils/uint8-array-compact-protocol.js";
 import { planColumnPages } from "./page-planner.js";
 import { planDictionary } from "./dictionary-planner.js";
+import { decodeParquetSplitBlockBloomFilter, encodeParquetBloomFilterValue, encodeParquetSplitBlockBloomFilter } from "../../lib/parquet-bloom-filter.js";
+import { BoundaryOrder } from "../parquet-thrift/BoundaryOrder.js";
+import { EncryptionWithColumnKey, EncryptionWithFooterKey } from "../parquet-thrift/ColumnCryptoMetaData.js";
+import { EncryptionAlgorithm } from "../parquet-thrift/EncryptionAlgorithm.js";
+import { FileCryptoMetaData } from "../parquet-thrift/FileCryptoMetaData.js";
+import { PARQUET_MAGIC_ENCRYPTED } from "../../lib/constants.js";
+import { createParquetModuleAad, createParquetFooterSignature, encryptParquetModule } from "../../lib/parquet-encryption.js";
 /**
  * Parquet File Magic String
  */
 const PARQUET_MAGIC = 'PAR1';
 const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
+const PARQUET_MAGIC_ENCRYPTED_BYTES = encodeUtf8(PARQUET_MAGIC_ENCRYPTED);
 /**
  * Parquet File Format Version
  */
@@ -170,6 +180,18 @@ export class ParquetEnvelopeWriter {
     dictionary;
     columnDictionaries;
     dictionaryPageSizeLimit;
+    bloomFilter;
+    pageIndex;
+    writePageChecksums;
+    writeSizeStatistics;
+    writeStatistics;
+    sortingColumns;
+    /** Optional modular-encryption configuration for the footer. */
+    encryption;
+    /** Optional plaintext-footer signature configuration. */
+    footerSignature;
+    /** File identifier used for encrypted column-module AAD. */
+    encryptionFileUnique;
     constructor(schema, writeFn, closeFn, fileOffset, opts) {
         this.schema = schema;
         this.write = writeFn;
@@ -182,6 +204,40 @@ export class ParquetEnvelopeWriter {
         this.dictionary = opts.dictionary ?? 'auto';
         this.columnDictionaries = opts.columnDictionaries || {};
         this.dictionaryPageSizeLimit = opts.dictionaryPageSizeLimit ?? 1024 * 1024;
+        this.bloomFilter = opts.bloomFilter;
+        this.pageIndex = opts.pageIndex;
+        this.writePageChecksums = Boolean(opts.writePageChecksums);
+        this.writeSizeStatistics = Boolean(opts.writeSizeStatistics);
+        this.writeStatistics = opts.writeStatistics;
+        this.sortingColumns = opts.sortingColumns;
+        const encryptionFileUnique = opts.encryption?.fileUnique
+            ? new Uint8Array(opts.encryption.fileUnique)
+            : undefined;
+        const footerSignatureFileUnique = opts.footerSignature?.fileUnique
+            ? new Uint8Array(opts.footerSignature.fileUnique)
+            : undefined;
+        if (encryptionFileUnique &&
+            footerSignatureFileUnique &&
+            !encryptionFileUnique.every((value, index) => value === footerSignatureFileUnique[index])) {
+            throw new Error('Parquet encryption and footer signature file_unique values must match');
+        }
+        const sharedFileUnique = encryptionFileUnique ??
+            footerSignatureFileUnique ??
+            (opts.encryption || opts.footerSignature ? createFileUnique() : undefined);
+        this.encryption = opts.encryption
+            ? { ...opts.encryption, fileUnique: new Uint8Array(sharedFileUnique) }
+            : undefined;
+        this.footerSignature = opts.footerSignature
+            ? {
+                ...opts.footerSignature,
+                keyMetadata: new Uint8Array(opts.footerSignature.keyMetadata),
+                aadPrefix: opts.footerSignature.aadPrefix
+                    ? new Uint8Array(opts.footerSignature.aadPrefix)
+                    : undefined,
+                fileUnique: new Uint8Array(sharedFileUnique)
+            }
+            : undefined;
+        this.encryptionFileUnique = sharedFileUnique;
     }
     writeSection(buf) {
         this.offset += buf.length;
@@ -191,7 +247,7 @@ export class ParquetEnvelopeWriter {
      * Encode the parquet file header
      */
     writeHeader() {
-        return this.writeSection(PARQUET_MAGIC_BYTES);
+        return this.writeSection(this.encryption && !this.footerSignature ? PARQUET_MAGIC_ENCRYPTED_BYTES : PARQUET_MAGIC_BYTES);
     }
     /**
      * Encode a parquet row group. The records object should be created using the
@@ -204,7 +260,16 @@ export class ParquetEnvelopeWriter {
             useDataPageV2: this.useDataPageV2,
             dictionary: this.dictionary,
             columnDictionaries: this.columnDictionaries,
-            dictionaryPageSizeLimit: this.dictionaryPageSizeLimit
+            dictionaryPageSizeLimit: this.dictionaryPageSizeLimit,
+            bloomFilter: this.bloomFilter,
+            pageIndex: this.pageIndex,
+            writePageChecksums: this.writePageChecksums,
+            writeSizeStatistics: this.writeSizeStatistics,
+            writeStatistics: this.writeStatistics,
+            sortingColumns: this.sortingColumns,
+            encryption: this.encryption,
+            encryptionFileUnique: this.encryptionFileUnique,
+            rowGroupOrdinal: this.rowGroups.length
         });
         this.rowCount += records.rowCount;
         this.rowGroups.push(rgroup.metadata);
@@ -213,12 +278,21 @@ export class ParquetEnvelopeWriter {
     /**
      * Write the parquet file footer
      */
-    writeFooter(userMetadata) {
+    async writeFooter(userMetadata) {
         if (!userMetadata) {
             // tslint:disable-next-line:no-parameter-reassignment
             userMetadata = {};
         }
-        return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
+        if (this.footerSignature) {
+            await this.writeSection(await encodeSignedFooter(this.schema, this.rowCount, this.rowGroups, userMetadata, this.footerSignature));
+            return;
+        }
+        const footer = encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata);
+        if (!this.encryption) {
+            await this.writeSection(footer);
+            return;
+        }
+        await this.writeSection(await encodeEncryptedFooter(footer, this.encryption));
     }
     /**
      * Set the parquet data page size. The data page size controls the maximum
@@ -276,7 +350,7 @@ function encodeValues(type, encoding, values, opts) {
 /**
  * Encode a parquet data page
  */
-async function encodeDataPage(column, data, valueEncoding = column.encoding, valueBitWidth) {
+async function encodeDataPage(column, data, valueEncoding = column.encoding, valueBitWidth, writePageChecksums = false, statistics, int96AsTimestamp = false) {
     /* encode repetition and definition levels */
     let rLevelsBuf = new Uint8Array(0);
     if (column.rLevelMax > 0) {
@@ -295,7 +369,8 @@ async function encodeDataPage(column, data, valueEncoding = column.encoding, val
     /* encode values */
     const valuesBuf = encodeValues(column.primitiveType, valueEncoding, data.values, {
         typeLength: column.typeLength,
-        bitWidth: valueBitWidth ?? column.typeLength
+        bitWidth: valueBitWidth ?? column.typeLength,
+        int96AsTimestamp
     });
     const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
     // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
@@ -307,10 +382,12 @@ async function encodeDataPage(column, data, valueEncoding = column.encoding, val
             num_values: data.count,
             encoding: Encoding[valueEncoding],
             definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],
-            repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]
+            repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING]
+            statistics
         }),
         uncompressed_page_size: dataBuf.length,
-        compressed_page_size: compressedBuf.length
+        compressed_page_size: compressedBuf.length,
+        crc: writePageChecksums ? crc32(compressedBuf) : undefined
     });
     /* concat page header, repetition and definition levels and values */
     const headerBuf = serializeThrift(header);
@@ -320,11 +397,12 @@ async function encodeDataPage(column, data, valueEncoding = column.encoding, val
 /**
  * Encode a parquet data page (v2)
  */
-async function encodeDataPageV2(column, data, rowCount, valueEncoding = column.encoding, valueBitWidth) {
+async function encodeDataPageV2(column, data, rowCount, valueEncoding = column.encoding, valueBitWidth, writePageChecksums = false, statistics, int96AsTimestamp = false) {
     /* encode values */
     const valuesBuf = encodeValues(column.primitiveType, valueEncoding, data.values, {
         typeLength: column.typeLength,
-        bitWidth: valueBitWidth ?? column.typeLength
+        bitWidth: valueBitWidth ?? column.typeLength,
+        int96AsTimestamp
     });
     // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
     const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
@@ -353,10 +431,14 @@ async function encodeDataPageV2(column, data, rowCount, valueEncoding = column.e
             encoding: Encoding[valueEncoding],
             definition_levels_byte_length: dLevelsBuf.length,
             repetition_levels_byte_length: rLevelsBuf.length,
-            is_compressed: column.compression !== 'UNCOMPRESSED'
+            is_compressed: column.compression !== 'UNCOMPRESSED',
+            statistics
         }),
         uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
-        compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
+        compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length,
+        crc: writePageChecksums
+            ? crc32(concatUint8Arrays([rLevelsBuf, dLevelsBuf, compressedBuf]))
+            : undefined
     });
     /* concat page header, repetition and definition levels and values */
     const headerBuf = serializeThrift(header);
@@ -364,9 +446,10 @@ async function encodeDataPageV2(column, data, rowCount, valueEncoding = column.e
     return { header, headerSize: headerBuf.length, page };
 }
 /** Encodes one chunk-wide PLAIN dictionary page. */
-async function encodeDictionaryPage(column, dictionaryValues) {
+async function encodeDictionaryPage(column, dictionaryValues, writePageChecksums = false, int96AsTimestamp = false) {
     const valuesBuffer = encodeValues(column.primitiveType, 'PLAIN', dictionaryValues, {
-        typeLength: column.typeLength
+        typeLength: column.typeLength,
+        int96AsTimestamp
     });
     const compressedBuffer = await Compression.deflate(column.compression, valuesBuffer);
     const header = new PageHeader({
@@ -376,7 +459,8 @@ async function encodeDictionaryPage(column, dictionaryValues) {
             encoding: Encoding.PLAIN
         }),
         uncompressed_page_size: valuesBuffer.length,
-        compressed_page_size: compressedBuffer.length
+        compressed_page_size: compressedBuffer.length,
+        crc: writePageChecksums ? crc32(compressedBuffer) : undefined
     });
     const headerBuffer = serializeThrift(header);
     return {
@@ -385,14 +469,64 @@ async function encodeDictionaryPage(column, dictionaryValues) {
         page: concatUint8Arrays([headerBuffer, compressedBuffer])
     };
 }
+/** Computes native Parquet geospatial statistics for a GEOMETRY or GEOGRAPHY column chunk. */
+function createGeospatialStatistics(column, values) {
+    const logicalType = column.logicalType?.type || column.originalType;
+    if (logicalType !== 'GEOMETRY' && logicalType !== 'GEOGRAPHY')
+        return undefined;
+    const geometryTypes = new Set();
+    let bbox;
+    // Vertex bounds are not conservative for GEOGRAPHY: non-linear edge algorithms can reach
+    // extrema between vertices. Until edge-aware geodesic bounds are available, omit the bbox while
+    // retaining the useful geometry-type statistics.
+    const canWriteBoundingBox = logicalType === 'GEOMETRY';
+    for (const value of values) {
+        if (!(value instanceof ArrayBuffer) && !ArrayBuffer.isView(value)) {
+            throw new Error(`${logicalType} columns must contain WKB binary values`);
+        }
+        const statistics = getWKBGeometryStatistics(value);
+        geometryTypes.add(statistics.geometryType);
+        if (canWriteBoundingBox) {
+            bbox = mergeGeometryBoundingBoxes(bbox, statistics.bbox);
+        }
+    }
+    return new GeospatialStatistics({
+        bbox: bbox ? new BoundingBox(bbox) : undefined,
+        geospatial_types: [...geometryTypes].sort((left, right) => left - right)
+    });
+}
+/** Merges finite per-dimension bounds without inventing absent Z or M dimensions. */
+function mergeGeometryBoundingBoxes(target, source) {
+    if (!source)
+        return target;
+    if (!target)
+        return { ...source };
+    target.xmin = Math.min(target.xmin, source.xmin);
+    target.xmax = Math.max(target.xmax, source.xmax);
+    target.ymin = Math.min(target.ymin, source.ymin);
+    target.ymax = Math.max(target.ymax, source.ymax);
+    mergeOptionalBounds(target, source, 'zmin', Math.min);
+    mergeOptionalBounds(target, source, 'zmax', Math.max);
+    mergeOptionalBounds(target, source, 'mmin', Math.min);
+    mergeOptionalBounds(target, source, 'mmax', Math.max);
+    return target;
+}
+/** Merges one optional bound while preserving a dimension absent from every value. */
+function mergeOptionalBounds(target, source, key, merge) {
+    const sourceValue = source[key];
+    if (sourceValue === undefined)
+        return;
+    target[key] = target[key] === undefined ? sourceValue : merge(target[key], sourceValue);
+}
 /**
  * Encode an array of values into a parquet column chunk
  */
-async function encodeColumnChunk(column, buffer, offset, opts) {
+async function encodeColumnChunk(column, buffer, offset, opts, rowGroupOrdinal, columnOrdinal) {
     const data = buffer.columnData[column.path.join()];
     const baseOffset = (opts.baseOffset || 0) + offset;
     /* encode data page(s) */
-    const pages = [];
+    const pageRecords = [];
+    const pageLocations = [];
     // tslint:disable-next-line:variable-name
     let total_uncompressed_size = 0;
     // tslint:disable-next-line:variable-name
@@ -401,12 +535,18 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
     const dictionaryPolicy = opts.columnDictionaries?.[column.path[0]] ?? opts.dictionary ?? 'auto';
     const dictionaryPlan = planDictionary(column, Array.from(data.values), dictionaryPolicy, opts.dictionaryPageSizeLimit ?? 1024 * 1024);
     if (dictionaryPlan) {
-        const result = await encodeDictionaryPage(column, dictionaryPlan.values);
-        pages.push(result.page);
+        const result = await encodeDictionaryPage(column, dictionaryPlan.values, opts.writePageChecksums, opts.int96AsTimestamp);
+        pageRecords.push({ page: result.page, headerSize: result.headerSize, dictionary: true });
         total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
         total_compressed_size += result.header.compressed_page_size + result.headerSize;
     }
-    const valueEncoding = dictionaryPlan ? 'RLE_DICTIONARY' : column.encoding;
+    const valueEncoding = dictionaryPlan
+        ? column.encoding === 'PLAIN_DICTIONARY'
+            ? 'PLAIN_DICTIONARY'
+            : 'RLE_DICTIONARY'
+        : column.encoding === 'PLAIN_DICTIONARY' || column.encoding === 'RLE_DICTIONARY'
+            ? 'PLAIN'
+            : column.encoding;
     let dictionaryIndexOffset = 0;
     for (const plannedPage of plannedPages) {
         const pageData = dictionaryPlan
@@ -417,14 +557,51 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
             : plannedPage.data;
         dictionaryIndexOffset += plannedPage.data.values.length;
         const result = opts.useDataPageV2
-            ? await encodeDataPageV2(column, pageData, plannedPage.rowCount, valueEncoding, dictionaryPlan?.bitWidth)
-            : await encodeDataPage(column, pageData, valueEncoding, dictionaryPlan?.bitWidth);
-        pages.push(result.page);
+            ? await encodeDataPageV2(column, pageData, plannedPage.rowCount, valueEncoding, dictionaryPlan?.bitWidth, opts.writePageChecksums, isStatisticsEnabled(opts.writeStatistics, column)
+                ? createColumnStatistics(column, plannedPage.data)
+                : undefined, opts.int96AsTimestamp)
+            : await encodeDataPage(column, pageData, valueEncoding, dictionaryPlan?.bitWidth, opts.writePageChecksums, isStatisticsEnabled(opts.writeStatistics, column)
+                ? createColumnStatistics(column, plannedPage.data)
+                : undefined, opts.int96AsTimestamp);
+        pageRecords.push({ page: result.page, headerSize: result.headerSize, dictionary: false });
         total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
         total_compressed_size += result.header.compressed_page_size + result.headerSize;
     }
+    const encryption = opts.encryption;
+    const encryptedColumn = Boolean(encryption && shouldEncryptColumn(encryption, column));
+    const columnKeyMetadata = encryption ? getColumnKeyMetadata(encryption, column) : undefined;
+    const pages = encryptedColumn
+        ? await encryptParquetColumnPages(pageRecords, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : pageRecords.map(record => record.page);
+    let pageOffset = 0;
+    let dataPageIndex = 0;
+    let firstRowIndex = 0;
+    for (const [pageIndex, page] of pages.entries()) {
+        if (!pageRecords[pageIndex].dictionary) {
+            pageLocations.push(new PageLocation({
+                offset: baseOffset + pageOffset,
+                compressed_page_size: page.length,
+                first_row_index: firstRowIndex
+            }));
+            firstRowIndex += plannedPages[dataPageIndex++].rowCount;
+        }
+        pageOffset += page.length;
+    }
+    total_compressed_size = pages.reduce((total, page) => total + page.length, 0);
     const pagesBuf = concatUint8Arrays(pages);
     // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
+    const bloomFilterEnabled = opts.bloomFilter === true || opts.bloomFilter?.[column.path[0]] === true;
+    const bloomFilter = bloomFilterEnabled ? createColumnBloomFilter(column, data.values) : undefined;
+    const encodedBloomFilter = bloomFilter && encryptedColumn
+        ? await encryptParquetBloomFilter(bloomFilter, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : bloomFilter;
+    const pageIndexEnabled = opts.pageIndex === true || opts.pageIndex?.[column.path[0]] === true;
+    const pageIndexes = pageIndexEnabled
+        ? createColumnPageIndexes(column, plannedPages, pageLocations, opts.int96AsTimestamp)
+        : undefined;
+    const encodedPageIndexes = pageIndexes && encryptedColumn
+        ? await encryptParquetPageIndexes(pageIndexes, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : pageIndexes;
     /* prepare metadata header */
     const metadata = new ColumnMetaData({
         path_in_schema: column.path,
@@ -451,7 +628,14 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
         total_uncompressed_size: int64(total_uncompressed_size), //  : pagesBuf.length,
         total_compressed_size: int64(total_compressed_size),
         type: Type[column.primitiveType],
-        codec: CompressionCodec[column.compression]
+        codec: CompressionCodec[column.compression],
+        bloom_filter_offset: encodedBloomFilter ? int64(baseOffset + pagesBuf.length) : undefined,
+        bloom_filter_length: encodedBloomFilter?.length,
+        geospatial_statistics: createGeospatialStatistics(column, data.values),
+        size_statistics: opts.writeSizeStatistics ? createSizeStatistics(column, data) : undefined,
+        statistics: isStatisticsEnabled(opts.writeStatistics, column)
+            ? createColumnStatistics(column, data)
+            : undefined
     });
     /* list encodings */
     metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);
@@ -459,9 +643,390 @@ async function encodeColumnChunk(column, buffer, offset, opts) {
         metadata.encodings.push(Encoding.PLAIN);
     metadata.encodings.push(Encoding[valueEncoding]);
     /* concat metadata header and data pages */
-    const metadataOffset = baseOffset + pagesBuf.length;
-    const body = concatUint8Arrays([pagesBuf, serializeThrift(metadata)]);
-    return { body, metadata, metadataOffset };
+    const metadataEncoded = serializeThrift(metadata);
+    const metadataSection = encryptedColumn ? undefined : metadataEncoded;
+    const metadataOffset = baseOffset + pagesBuf.length + (encodedBloomFilter?.length || 0);
+    const offsetIndexOffset = encodedPageIndexes
+        ? metadataOffset + (metadataSection?.length || 0)
+        : undefined;
+    const columnIndexOffset = encodedPageIndexes?.columnIndex
+        ? metadataOffset + (metadataSection?.length || 0) + encodedPageIndexes.offsetIndex.length
+        : undefined;
+    const body = concatUint8Arrays([
+        pagesBuf,
+        ...(encodedBloomFilter ? [encodedBloomFilter] : []),
+        ...(metadataSection ? [metadataSection] : []),
+        ...(encodedPageIndexes
+            ? [
+                encodedPageIndexes.offsetIndex,
+                ...(encodedPageIndexes.columnIndex ? [encodedPageIndexes.columnIndex] : [])
+            ]
+            : [])
+    ]);
+    return {
+        body,
+        metadata,
+        metadataOffset,
+        offsetIndexOffset,
+        offsetIndexLength: encodedPageIndexes?.offsetIndex.length,
+        columnIndexOffset,
+        columnIndexLength: encodedPageIndexes?.columnIndex?.length,
+        ...(encryptedColumn
+            ? {
+                encryptedColumnMetadata: await encryptParquetModule(metadataEncoded, {
+                    algorithm: encryption.algorithm ?? 'AES_GCM_V1',
+                    aad: createParquetModuleAad(encryption.aadPrefix, opts.encryptionFileUnique, 'column-metadata', opts.rowGroupOrdinal ?? 0, columnOrdinal),
+                    keyMetadata: columnKeyMetadata ?? encryption.keyMetadata,
+                    keyRetriever: encryption.keyRetriever
+                }),
+                cryptoMetadata: new ColumnCryptoMetaData({
+                    ...(columnKeyMetadata
+                        ? {
+                            ENCRYPTION_WITH_COLUMN_KEY: new EncryptionWithColumnKey({
+                                path_in_schema: [...column.path],
+                                key_metadata: new Uint8Array(columnKeyMetadata)
+                            })
+                        }
+                        : { ENCRYPTION_WITH_FOOTER_KEY: new EncryptionWithFooterKey() })
+                })
+            }
+            : {})
+    };
+}
+/** Returns whether a writer column should use modular encryption. */
+function shouldEncryptColumn(options, column) {
+    return Boolean(options.columnKeyMetadata?.[column.path[0]] ||
+        options.encryptColumns === true ||
+        options.encryptColumns?.[column.path[0]] === true);
+}
+/** Returns a defensive copy of the key metadata assigned to one top-level column. */
+function getColumnKeyMetadata(options, column) {
+    const keyMetadata = options.columnKeyMetadata?.[column.path[0]];
+    return keyMetadata ? new Uint8Array(keyMetadata) : undefined;
+}
+/** Adds the four-byte module length required by encrypted page and Bloom-filter modules. */
+function prefixEncryptedModule(module) {
+    const prefixed = new Uint8Array(module.byteLength + 4);
+    writeUInt32LE(prefixed, module.byteLength, 0);
+    prefixed.set(module, 4);
+    return prefixed;
+}
+/** Encrypts one column's dictionary and data pages using the Parquet module AAD rules. */
+async function encryptParquetColumnPages(pageRecords, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const encryptedPages = [];
+    let dataPageOrdinal = 0;
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    for (const pageRecord of pageRecords) {
+        const moduleType = pageRecord.dictionary ? 'dictionary-page' : 'data-page';
+        const headerModuleType = pageRecord.dictionary ? 'dictionary-page-header' : 'data-page-header';
+        const header = await encryptParquetModule(pageRecord.page.subarray(0, pageRecord.headerSize), {
+            algorithm,
+            aad: createParquetModuleAad(options.aadPrefix, fileUnique, headerModuleType, rowGroupOrdinal, columnOrdinal, pageRecord.dictionary ? undefined : dataPageOrdinal),
+            keyMetadata: keyMetadata ?? options.keyMetadata,
+            keyRetriever: options.keyRetriever
+        });
+        const body = await encryptParquetModule(pageRecord.page.subarray(pageRecord.headerSize), {
+            algorithm,
+            aad: createParquetModuleAad(options.aadPrefix, fileUnique, moduleType, rowGroupOrdinal, columnOrdinal, pageRecord.dictionary ? undefined : dataPageOrdinal),
+            keyMetadata: keyMetadata ?? options.keyMetadata,
+            keyRetriever: options.keyRetriever,
+            page: true
+        });
+        encryptedPages.push(concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(body)]));
+        if (!pageRecord.dictionary)
+            dataPageOrdinal++;
+    }
+    return encryptedPages;
+}
+/** Encrypts a split-block Bloom filter header and bitset as independent Parquet modules. */
+async function encryptParquetBloomFilter(bloomFilter, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const parsed = decodeParquetSplitBlockBloomFilter(bloomFilter);
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const keyOptions = {
+        algorithm,
+        keyMetadata: keyMetadata ?? options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    };
+    const header = await encryptParquetModule(bloomFilter.subarray(0, parsed.headerByteLength), {
+        ...keyOptions,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'bloom-filter-header', rowGroupOrdinal, columnOrdinal)
+    });
+    const bitset = await encryptParquetModule(parsed.bitset, {
+        ...keyOptions,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'bloom-filter-bitset', rowGroupOrdinal, columnOrdinal)
+    });
+    return concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(bitset)]);
+}
+/** Encrypts column and offset indexes independently while preserving their footer offsets. */
+async function encryptParquetPageIndexes(pageIndexes, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const encryptIndex = (bytes, module) => encryptParquetModule(bytes, {
+        algorithm,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, module, rowGroupOrdinal, columnOrdinal),
+        keyMetadata: keyMetadata ?? options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    });
+    return {
+        offsetIndex: await encryptIndex(pageIndexes.offsetIndex, 'offset-index'),
+        columnIndex: pageIndexes.columnIndex
+            ? await encryptIndex(pageIndexes.columnIndex, 'column-index')
+            : undefined
+    };
+}
+/** Builds page indexes for primitive leaves, including nested and repeated leaves. */
+function createColumnPageIndexes(column, plannedPages, pageLocations, int96AsTimestamp = false) {
+    if (!column.primitiveType ||
+        !isPageIndexPhysicalType(column.primitiveType) ||
+        plannedPages.length !== pageLocations.length ||
+        plannedPages.length === 0) {
+        return undefined;
+    }
+    const offsetIndex = serializeThrift(new OffsetIndex({ page_locations: pageLocations }));
+    if (!supportsStatisticsSortOrder(column)) {
+        return { offsetIndex };
+    }
+    const nullPages = [];
+    const minValues = [];
+    const maxValues = [];
+    // A repeated leaf's definition-level count is not a row-level null count.
+    // Omit null_counts for those leaves so readers do not interpret entry counts
+    // as null counts while still benefiting from min/max page pruning.
+    const nullCounts = column.rLevelMax === 0 ? [] : undefined;
+    for (const plannedPage of plannedPages) {
+        const values = plannedPage.data.values;
+        const nullCount = plannedPage.data.count - values.length;
+        nullCounts?.push(nullCount);
+        if (values.length === 0) {
+            nullPages.push(true);
+            minValues.push(new Uint8Array(0));
+            maxValues.push(new Uint8Array(0));
+            continue;
+        }
+        const bounds = getPageBounds(values, column.primitiveType, column.typeLength, int96AsTimestamp);
+        if (!bounds)
+            return undefined;
+        nullPages.push(false);
+        minValues.push(bounds.min);
+        maxValues.push(bounds.max);
+    }
+    return {
+        offsetIndex,
+        columnIndex: serializeThrift(new ColumnIndex({
+            null_pages: nullPages,
+            min_values: minValues,
+            max_values: maxValues,
+            boundary_order: BoundaryOrder.UNORDERED,
+            null_counts: nullCounts
+        }))
+    };
+}
+function isPageIndexPhysicalType(physicalType) {
+    return (physicalType === 'BOOLEAN' ||
+        physicalType === 'INT32' ||
+        physicalType === 'INT64' ||
+        physicalType === 'INT96' ||
+        physicalType === 'FLOAT' ||
+        physicalType === 'DOUBLE' ||
+        physicalType === 'BYTE_ARRAY' ||
+        physicalType === 'FIXED_LEN_BYTE_ARRAY');
+}
+function getPageBounds(values, physicalType, typeLength, int96AsTimestamp = false) {
+    if (physicalType === 'INT96' && !int96AsTimestamp)
+        return undefined;
+    const normalizedValues = Array.from(values, value => normalizePageIndexValue(value, physicalType));
+    if (normalizedValues.some(value => value === undefined))
+        return undefined;
+    const comparableValues = normalizedValues;
+    let min = comparableValues[0];
+    let max = comparableValues[0];
+    for (const value of comparableValues.slice(1)) {
+        if (comparePageIndexValues(value, min, physicalType) < 0)
+            min = value;
+        if (comparePageIndexValues(value, max, physicalType) > 0)
+            max = value;
+    }
+    try {
+        return {
+            min: encodePageIndexValue(min, physicalType, typeLength, int96AsTimestamp),
+            max: encodePageIndexValue(max, physicalType, typeLength, int96AsTimestamp)
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function encodePageIndexValue(value, physicalType, typeLength, int96AsTimestamp) {
+    if (physicalType === 'INT96') {
+        if (!int96AsTimestamp) {
+            throw new Error('INT96 page indexes require timestamp mode');
+        }
+        return PARQUET_CODECS.PLAIN.encodeValues('INT96', [value], { int96AsTimestamp });
+    }
+    return encodeParquetBloomFilterValue(value, physicalType, typeLength);
+}
+function normalizePageIndexValue(value, physicalType) {
+    if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
+        if (typeof value === 'string')
+            return new TextEncoder().encode(value);
+        if (value instanceof ArrayBuffer)
+            return new Uint8Array(value);
+        if (ArrayBuffer.isView(value)) {
+            return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
+        }
+        return undefined;
+    }
+    if (typeof value === 'boolean' ||
+        typeof value === 'number' ||
+        typeof value === 'bigint' ||
+        typeof value === 'string') {
+        return value;
+    }
+    return undefined;
+}
+function comparePageIndexValues(left, right, physicalType) {
+    if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
+        const leftBytes = typeof left === 'string' ? new TextEncoder().encode(left) : left;
+        const rightBytes = typeof right === 'string' ? new TextEncoder().encode(right) : right;
+        if (!(leftBytes instanceof Uint8Array) || !(rightBytes instanceof Uint8Array))
+            return 0;
+        const length = Math.min(leftBytes.length, rightBytes.length);
+        for (let index = 0; index < length; index++) {
+            if (leftBytes[index] !== rightBytes[index]) {
+                return leftBytes[index] < rightBytes[index] ? -1 : 1;
+            }
+        }
+        return leftBytes.length - rightBytes.length;
+    }
+    if (typeof left === 'boolean' && typeof right === 'boolean')
+        return Number(left) - Number(right);
+    if (physicalType !== 'INT64' && physicalType !== 'INT96') {
+        const leftNumber = Number(left);
+        const rightNumber = Number(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+    try {
+        const leftNumber = BigInt(left);
+        const rightNumber = BigInt(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+    catch {
+        const leftNumber = Number(left);
+        const rightNumber = Number(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+}
+/** Builds a Bloom filter for writer-supported physical scalar columns. */
+function createColumnBloomFilter(column, values) {
+    const physicalType = column.primitiveType;
+    if (physicalType !== 'BOOLEAN' &&
+        physicalType !== 'INT32' &&
+        physicalType !== 'INT64' &&
+        physicalType !== 'FLOAT' &&
+        physicalType !== 'DOUBLE' &&
+        physicalType !== 'BYTE_ARRAY' &&
+        physicalType !== 'FIXED_LEN_BYTE_ARRAY') {
+        return undefined;
+    }
+    const bloomValues = values.map(value => {
+        if (value instanceof ArrayBuffer)
+            return new Uint8Array(value);
+        if (ArrayBuffer.isView(value)) {
+            return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
+        }
+        return value;
+    });
+    return encodeParquetSplitBlockBloomFilter(bloomValues, physicalType, column.typeLength);
+}
+/** Builds optional size statistics from one shredded column chunk. */
+function createSizeStatistics(column, data) {
+    const repetitionLevelHistogram = createLevelHistogram(data.rlevels, column.rLevelMax);
+    const definitionLevelHistogram = createLevelHistogram(data.dlevels, column.dLevelMax);
+    let unencodedByteArrayDataBytes = 0;
+    if (column.primitiveType === 'BYTE_ARRAY') {
+        for (const value of data.values) {
+            if (typeof value === 'string') {
+                unencodedByteArrayDataBytes += encodeUtf8(value).byteLength;
+            }
+            else if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
+                unencodedByteArrayDataBytes += value.byteLength;
+            }
+        }
+    }
+    return new SizeStatistics({
+        unencoded_byte_array_data_bytes: column.primitiveType === 'BYTE_ARRAY' ? unencodedByteArrayDataBytes : undefined,
+        repetition_level_histogram: repetitionLevelHistogram,
+        definition_level_histogram: definitionLevelHistogram
+    });
+}
+/** Returns whether standard column statistics are enabled for one top-level field. */
+function isStatisticsEnabled(option, column) {
+    return option === true || option?.[column.path[0]] === true;
+}
+/** Builds conservative standard min/max/null-count statistics for one column chunk. */
+function createColumnStatistics(column, data) {
+    if (!column.primitiveType ||
+        !isPageIndexPhysicalType(column.primitiveType) ||
+        !supportsStatisticsSortOrder(column)) {
+        return undefined;
+    }
+    const statistics = new Statistics({
+        null_count: column.rLevelMax === 0 ? data.count - data.values.length : undefined,
+        is_min_value_exact: true,
+        is_max_value_exact: true
+    });
+    if (data.values.length === 0)
+        return statistics;
+    const bounds = getPageBounds(data.values, column.primitiveType, column.typeLength);
+    if (!bounds)
+        return undefined;
+    statistics.min_value = bounds.min;
+    statistics.max_value = bounds.max;
+    return statistics;
+}
+/** Returns whether the physical representation preserves the logical sort order. */
+function supportsStatisticsSortOrder(column) {
+    const logicalType = column.logicalType?.type || column.originalType;
+    if (logicalType === 'FLOAT16')
+        return false;
+    if (logicalType === 'DECIMAL' || logicalType?.startsWith('DECIMAL_')) {
+        return column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY';
+    }
+    if (logicalType === 'INTEGER' || logicalType?.startsWith('UINT_')) {
+        return column.logicalType?.isSigned !== false && !logicalType.startsWith('UINT_');
+    }
+    return true;
+}
+/** Counts the occurrences of each definition or repetition level. */
+function createLevelHistogram(levels, maximumLevel) {
+    const histogram = new Array(maximumLevel + 1).fill(0);
+    for (const level of levels) {
+        if (level >= 0 && level <= maximumLevel)
+            histogram[level]++;
+    }
+    return histogram;
+}
+/** Converts writer-facing sort keys into Parquet leaf-column indexes. */
+function createSortingColumns(schema, sortingColumns) {
+    if (!sortingColumns?.length)
+        return undefined;
+    const leafFields = schema.fieldList.filter(field => !field.isNested);
+    const usedColumnIndexes = new Set();
+    return sortingColumns.map(sortKey => {
+        const columnIndex = leafFields.findIndex(field => (field.path.length === 1 && field.name === sortKey.column) ||
+            (field.path.length > 1 && field.path.join('.') === sortKey.column));
+        if (columnIndex < 0) {
+            throw new Error(`Unknown Parquet sorting column ${sortKey.column}`);
+        }
+        if (usedColumnIndexes.has(columnIndex)) {
+            throw new Error(`Duplicate Parquet sorting column ${sortKey.column}`);
+        }
+        usedColumnIndexes.add(columnIndex);
+        return new SortingColumn({
+            column_idx: columnIndex,
+            descending: Boolean(sortKey.descending),
+            nulls_first: Boolean(sortKey.nullsFirst)
+        });
+    });
 }
 /**
  * Encode a list of column values into a parquet row group
@@ -470,17 +1035,28 @@ async function encodeRowGroup(schema, data, opts) {
     const metadata = new RowGroup({
         num_rows: int64(data.rowCount),
         columns: [],
-        total_byte_size: int64(0)
+        total_byte_size: int64(0),
+        sorting_columns: createSortingColumns(schema, opts.sortingColumns)
     });
     let body = new Uint8Array(0);
     for (const field of schema.fieldList) {
         if (field.isNested) {
             continue; // eslint-disable-line no-continue
         }
-        const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
+        const cchunkData = await encodeColumnChunk(field, data, body.length, opts, opts.rowGroupOrdinal ?? 0, metadata.columns.length);
         const cchunk = new ColumnChunk({
             file_offset: int64(cchunkData.metadataOffset),
-            meta_data: cchunkData.metadata
+            meta_data: cchunkData.encryptedColumnMetadata ? undefined : cchunkData.metadata,
+            offset_index_offset: cchunkData.offsetIndexOffset === undefined
+                ? undefined
+                : int64(cchunkData.offsetIndexOffset),
+            offset_index_length: cchunkData.offsetIndexLength,
+            column_index_offset: cchunkData.columnIndexOffset === undefined
+                ? undefined
+                : int64(cchunkData.columnIndexOffset),
+            column_index_length: cchunkData.columnIndexLength,
+            crypto_metadata: cchunkData.cryptoMetadata,
+            encrypted_column_metadata: cchunkData.encryptedColumnMetadata
         });
         metadata.columns.push(cchunk);
         metadata.total_byte_size = int64(Number(metadata.total_byte_size) + cchunkData.body.length);
@@ -491,14 +1067,38 @@ async function encodeRowGroup(schema, data, opts) {
 /**
  * Encode a parquet file metadata footer
  */
-function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
+function encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, footerSignature) {
+    const signatureAlgorithm = footerSignature?.algorithm ?? 'AES_GCM_V1';
+    const signatureFileUnique = footerSignature?.fileUnique
+        ? new Uint8Array(footerSignature.fileUnique)
+        : undefined;
     const metadata = new FileMetaData({
         version: PARQUET_VERSION,
         created_by: 'parquets',
         num_rows: int64(rowCount),
         row_groups: rowGroups,
         schema: [],
-        key_value_metadata: []
+        key_value_metadata: [],
+        encryption_algorithm: footerSignature
+            ? new EncryptionAlgorithm({
+                ...(signatureAlgorithm === 'AES_GCM_CTR_V1'
+                    ? {
+                        AES_GCM_CTR_V1: {
+                            aad_prefix: footerSignature.aadPrefix,
+                            aad_file_unique: signatureFileUnique
+                        }
+                    }
+                    : {
+                        AES_GCM_V1: {
+                            aad_prefix: footerSignature.aadPrefix,
+                            aad_file_unique: signatureFileUnique
+                        }
+                    })
+            })
+            : undefined,
+        footer_signing_key_metadata: footerSignature
+            ? new Uint8Array(footerSignature.keyMetadata)
+            : undefined
     });
     for (const key in userMetadata) {
         const kv = new KeyValue({
@@ -542,13 +1142,70 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
         }
         metadata.schema.push(schemaElem);
     }
-    const metadataEncoded = serializeThrift(metadata);
+    return serializeThrift(metadata);
+}
+/** Encodes an unsigned plaintext footer with its standard Parquet trailer. */
+function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
+    const metadataEncoded = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata);
     const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
     footerEncoded.set(metadataEncoded);
     writeUInt32LE(footerEncoded, metadataEncoded.length, metadataEncoded.length);
     footerEncoded.set(PARQUET_MAGIC_BYTES, metadataEncoded.length + 4);
     return footerEncoded;
 }
+/** Encodes a plaintext footer with its trailing 28-byte authenticated signature. */
+async function encodeSignedFooter(schema, rowCount, rowGroups, userMetadata, options) {
+    const metadata = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, options);
+    const signature = await createParquetFooterSignature(metadata, options);
+    const footerEncoded = new Uint8Array(metadata.length + signature.length + 8);
+    footerEncoded.set(metadata);
+    footerEncoded.set(signature, metadata.length);
+    writeUInt32LE(footerEncoded, metadata.length + signature.length, metadata.length + signature.length);
+    footerEncoded.set(PARQUET_MAGIC_BYTES, metadata.length + signature.length + 4);
+    return footerEncoded;
+}
+/** Wraps a serialized footer in Parquet modular-encryption metadata and AES-GCM. */
+async function encodeEncryptedFooter(footer, options) {
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const fileUnique = options.fileUnique ? new Uint8Array(options.fileUnique) : createFileUnique();
+    if (fileUnique.byteLength !== 8) {
+        throw new Error(`Parquet encrypted footer file_unique must be 8 bytes`);
+    }
+    const algorithmParameters = {
+        aad_prefix: options.aadPrefix && new Uint8Array(options.aadPrefix),
+        aad_file_unique: fileUnique,
+        supply_aad_prefix: undefined
+    };
+    const encryptionAlgorithm = algorithm === 'AES_GCM_CTR_V1'
+        ? new EncryptionAlgorithm({ AES_GCM_CTR_V1: algorithmParameters })
+        : new EncryptionAlgorithm({ AES_GCM_V1: algorithmParameters });
+    const cryptoMetadata = serializeThrift(new FileCryptoMetaData({
+        encryption_algorithm: encryptionAlgorithm,
+        key_metadata: options.keyMetadata && new Uint8Array(options.keyMetadata)
+    }));
+    const footerMetadata = footer.subarray(0, footer.byteLength - 8);
+    const encryptedFooter = await encryptParquetModule(footerMetadata, {
+        algorithm,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'footer'),
+        keyMetadata: options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    });
+    const metadataSize = cryptoMetadata.byteLength + encryptedFooter.byteLength;
+    const trailer = new Uint8Array(8);
+    writeUInt32LE(trailer, metadataSize, 0);
+    trailer.set(encodeUtf8(PARQUET_MAGIC_ENCRYPTED), 4);
+    return concatUint8Arrays([cryptoMetadata, encryptedFooter, trailer]);
+}
+/** Generates the eight-byte file identifier required by Parquet module AAD. */
+function createFileUnique() {
+    const fileUnique = new Uint8Array(8);
+    const cryptoProvider = globalThis.crypto;
+    if (!cryptoProvider?.getRandomValues) {
+        throw new Error('Parquet encrypted footer requires Web Crypto random values');
+    }
+    cryptoProvider.getRandomValues(fileUnique);
+    return fileUnique;
+}
 /** Returns an explicit or inferred modern logical annotation for a writer field. */
 function getFieldLogicalType(field) {
     if (field.logicalType)