@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/dist/parquetjs/encoder/parquet-encoder.js

@@ -3,17 +3,31 @@
 // Copyright (c) vis.gl contributors
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
+import { getWKBGeometryStatistics } from '@loaders.gl/gis';
 import { PARQUET_CODECS } from "../codecs/index.js";
 import * as Compression from "../compression.js";
 import * as Shred from "../schema/shred.js";
-import { ColumnChunk, ColumnMetaData, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, Encoding, FieldRepetitionType, FileMetaData, KeyValue, PageHeader, PageType, RowGroup, SchemaElement, Type } from "../parquet-thrift/index.js";
+import { BsonType, BoundingBox, ColumnChunk, ColumnCryptoMetaData, ColumnIndex, ColumnMetaData, SizeStatistics, Statistics, SortingColumn, CompressionCodec, ConvertedType, DataPageHeader, DataPageHeaderV2, DateType, DecimalType, DictionaryPageHeader, EdgeInterpolationAlgorithm, Encoding, EnumType, FieldRepetitionType, FileMetaData, Float16Type, GeographyType, GeospatialStatistics, GeometryType, IntType, JsonType, KeyValue, ListType, LogicalType, MapType, MicroSeconds, MilliSeconds, NanoSeconds, NullType, PageHeader, PageEncodingStats, PageType, PageLocation, RowGroup, SchemaElement, StringType, TimeType, TimeUnit, TimestampType, Type, UUIDType, VariantType, OffsetIndex } from "../parquet-thrift/index.js";
 import { osopen, oswrite, osclose } from "../utils/file-utils.js";
 import { getBitWidth, serializeThrift } from "../utils/read-utils.js";
-import Int64 from 'node-int64';
+import { concatUint8Arrays, encodeUtf8, writeUInt32LE } from "../utils/binary-utils.js";
+import { crc32 } from "../utils/crc32.js";
+import { CompactInt64 } from "../utils/uint8-array-compact-protocol.js";
+import { planColumnPages } from "./page-planner.js";
+import { planDictionary } from "./dictionary-planner.js";
+import { decodeParquetSplitBlockBloomFilter, encodeParquetBloomFilterValue, encodeParquetSplitBlockBloomFilter } from "../../lib/parquet-bloom-filter.js";
+import { BoundaryOrder } from "../parquet-thrift/BoundaryOrder.js";
+import { EncryptionWithColumnKey, EncryptionWithFooterKey } from "../parquet-thrift/ColumnCryptoMetaData.js";
+import { EncryptionAlgorithm } from "../parquet-thrift/EncryptionAlgorithm.js";
+import { FileCryptoMetaData } from "../parquet-thrift/FileCryptoMetaData.js";
+import { PARQUET_MAGIC_ENCRYPTED } from "../../lib/constants.js";
+import { createParquetModuleAad, createParquetFooterSignature, encryptParquetModule } from "../../lib/parquet-encryption.js";
 /**
  * Parquet File Magic String
  */
 const PARQUET_MAGIC = 'PAR1';
+const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
+const PARQUET_MAGIC_ENCRYPTED_BYTES = encodeUtf8(PARQUET_MAGIC_ENCRYPTED);
 /**
  * Parquet File Format Version
  */
@@ -63,8 +77,7 @@ export class ParquetEncoder {
     constructor(schema, envelopeWriter, opts) {
         this.schema = schema;
         this.envelopeWriter = envelopeWriter;
-        // @ts-ignore Row buffer typings...
-        this.rowBuffer = {};
+        this.rowBuffer = schema.rowGroup();
         this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
         this.closed = false;
         this.userMetadata = {};
@@ -91,8 +104,8 @@ export class ParquetEncoder {
         }
         Shred.shredRecord(this.schema, row, this.rowBuffer);
         if (this.rowBuffer.rowCount >= this.rowGroupSize) {
-            // @ts-ignore
-            this.rowBuffer = {};
+            await this.envelopeWriter.writeRowGroup(this.rowBuffer);
+            this.rowBuffer = this.schema.rowGroup();
         }
     }
     /**
@@ -107,8 +120,8 @@ export class ParquetEncoder {
         }
         this.closed = true;
         if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
-            // @ts-ignore
-            this.rowBuffer = {};
+            await this.envelopeWriter.writeRowGroup(this.rowBuffer);
+            this.rowBuffer = this.schema.rowGroup();
         }
         await this.envelopeWriter.writeFooter(this.userMetadata);
         await this.envelopeWriter.close();
@@ -164,6 +177,21 @@ export class ParquetEnvelopeWriter {
     rowGroups;
     pageSize;
     useDataPageV2;
+    dictionary;
+    columnDictionaries;
+    dictionaryPageSizeLimit;
+    bloomFilter;
+    pageIndex;
+    writePageChecksums;
+    writeSizeStatistics;
+    writeStatistics;
+    sortingColumns;
+    /** Optional modular-encryption configuration for the footer. */
+    encryption;
+    /** Optional plaintext-footer signature configuration. */
+    footerSignature;
+    /** File identifier used for encrypted column-module AAD. */
+    encryptionFileUnique;
     constructor(schema, writeFn, closeFn, fileOffset, opts) {
         this.schema = schema;
         this.write = writeFn;
@@ -173,6 +201,43 @@ export class ParquetEnvelopeWriter {
         this.rowGroups = [];
         this.pageSize = opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE;
         this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
+        this.dictionary = opts.dictionary ?? 'auto';
+        this.columnDictionaries = opts.columnDictionaries || {};
+        this.dictionaryPageSizeLimit = opts.dictionaryPageSizeLimit ?? 1024 * 1024;
+        this.bloomFilter = opts.bloomFilter;
+        this.pageIndex = opts.pageIndex;
+        this.writePageChecksums = Boolean(opts.writePageChecksums);
+        this.writeSizeStatistics = Boolean(opts.writeSizeStatistics);
+        this.writeStatistics = opts.writeStatistics;
+        this.sortingColumns = opts.sortingColumns;
+        const encryptionFileUnique = opts.encryption?.fileUnique
+            ? new Uint8Array(opts.encryption.fileUnique)
+            : undefined;
+        const footerSignatureFileUnique = opts.footerSignature?.fileUnique
+            ? new Uint8Array(opts.footerSignature.fileUnique)
+            : undefined;
+        if (encryptionFileUnique &&
+            footerSignatureFileUnique &&
+            !encryptionFileUnique.every((value, index) => value === footerSignatureFileUnique[index])) {
+            throw new Error('Parquet encryption and footer signature file_unique values must match');
+        }
+        const sharedFileUnique = encryptionFileUnique ??
+            footerSignatureFileUnique ??
+            (opts.encryption || opts.footerSignature ? createFileUnique() : undefined);
+        this.encryption = opts.encryption
+            ? { ...opts.encryption, fileUnique: new Uint8Array(sharedFileUnique) }
+            : undefined;
+        this.footerSignature = opts.footerSignature
+            ? {
+                ...opts.footerSignature,
+                keyMetadata: new Uint8Array(opts.footerSignature.keyMetadata),
+                aadPrefix: opts.footerSignature.aadPrefix
+                    ? new Uint8Array(opts.footerSignature.aadPrefix)
+                    : undefined,
+                fileUnique: new Uint8Array(sharedFileUnique)
+            }
+            : undefined;
+        this.encryptionFileUnique = sharedFileUnique;
     }
     writeSection(buf) {
         this.offset += buf.length;
@@ -182,7 +247,7 @@ export class ParquetEnvelopeWriter {
      * Encode the parquet file header
      */
     writeHeader() {
-        return this.writeSection(Buffer.from(PARQUET_MAGIC));
+        return this.writeSection(this.encryption && !this.footerSignature ? PARQUET_MAGIC_ENCRYPTED_BYTES : PARQUET_MAGIC_BYTES);
     }
     /**
      * Encode a parquet row group. The records object should be created using the
@@ -192,7 +257,19 @@ export class ParquetEnvelopeWriter {
         const rgroup = await encodeRowGroup(this.schema, records, {
             baseOffset: this.offset,
             pageSize: this.pageSize,
-            useDataPageV2: this.useDataPageV2
+            useDataPageV2: this.useDataPageV2,
+            dictionary: this.dictionary,
+            columnDictionaries: this.columnDictionaries,
+            dictionaryPageSizeLimit: this.dictionaryPageSizeLimit,
+            bloomFilter: this.bloomFilter,
+            pageIndex: this.pageIndex,
+            writePageChecksums: this.writePageChecksums,
+            writeSizeStatistics: this.writeSizeStatistics,
+            writeStatistics: this.writeStatistics,
+            sortingColumns: this.sortingColumns,
+            encryption: this.encryption,
+            encryptionFileUnique: this.encryptionFileUnique,
+            rowGroupOrdinal: this.rowGroups.length
         });
         this.rowCount += records.rowCount;
         this.rowGroups.push(rgroup.metadata);
@@ -201,12 +278,21 @@ export class ParquetEnvelopeWriter {
     /**
      * Write the parquet file footer
      */
-    writeFooter(userMetadata) {
+    async writeFooter(userMetadata) {
         if (!userMetadata) {
             // tslint:disable-next-line:no-parameter-reassignment
             userMetadata = {};
         }
-        return this.writeSection(encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata));
+        if (this.footerSignature) {
+            await this.writeSection(await encodeSignedFooter(this.schema, this.rowCount, this.rowGroups, userMetadata, this.footerSignature));
+            return;
+        }
+        const footer = encodeFooter(this.schema, this.rowCount, this.rowGroups, userMetadata);
+        if (!this.encryption) {
+            await this.writeSection(footer);
+            return;
+        }
+        await this.writeSection(await encodeEncryptedFooter(footer, this.encryption));
     }
     /**
      * Set the parquet data page size. The data page size controls the maximum
@@ -264,16 +350,16 @@ function encodeValues(type, encoding, values, opts) {
 /**
  * Encode a parquet data page
  */
-async function encodeDataPage(column, data) {
+async function encodeDataPage(column, data, valueEncoding = column.encoding, valueBitWidth, writePageChecksums = false, statistics, int96AsTimestamp = false) {
     /* encode repetition and definition levels */
-    let rLevelsBuf = Buffer.alloc(0);
+    let rLevelsBuf = new Uint8Array(0);
     if (column.rLevelMax > 0) {
         rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
             bitWidth: getBitWidth(column.rLevelMax)
             // disableEnvelope: false
         });
     }
-    let dLevelsBuf = Buffer.alloc(0);
+    let dLevelsBuf = new Uint8Array(0);
     if (column.dLevelMax > 0) {
         dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
             bitWidth: getBitWidth(column.dLevelMax)
@@ -281,11 +367,12 @@ async function encodeDataPage(column, data) {
         });
     }
     /* encode values */
-    const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
+    const valuesBuf = encodeValues(column.primitiveType, valueEncoding, data.values, {
         typeLength: column.typeLength,
-        bitWidth: column.typeLength
+        bitWidth: valueBitWidth ?? column.typeLength,
+        int96AsTimestamp
     });
-    const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf]);
+    const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
     // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
     const compressedBuf = await Compression.deflate(column.compression, dataBuf);
     /* build page header */
@@ -293,38 +380,41 @@ async function encodeDataPage(column, data) {
         type: PageType.DATA_PAGE,
         data_page_header: new DataPageHeader({
             num_values: data.count,
-            encoding: Encoding[column.encoding],
+            encoding: Encoding[valueEncoding],
             definition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING],
-            repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING] // [PARQUET_RDLVL_ENCODING]
+            repetition_level_encoding: Encoding[PARQUET_RDLVL_ENCODING], // [PARQUET_RDLVL_ENCODING]
+            statistics
         }),
         uncompressed_page_size: dataBuf.length,
-        compressed_page_size: compressedBuf.length
+        compressed_page_size: compressedBuf.length,
+        crc: writePageChecksums ? crc32(compressedBuf) : undefined
     });
     /* concat page header, repetition and definition levels and values */
     const headerBuf = serializeThrift(header);
-    const page = Buffer.concat([headerBuf, compressedBuf]);
+    const page = concatUint8Arrays([headerBuf, compressedBuf]);
     return { header, headerSize: headerBuf.length, page };
 }
 /**
  * Encode a parquet data page (v2)
  */
-async function encodeDataPageV2(column, data, rowCount) {
+async function encodeDataPageV2(column, data, rowCount, valueEncoding = column.encoding, valueBitWidth, writePageChecksums = false, statistics, int96AsTimestamp = false) {
     /* encode values */
-    const valuesBuf = encodeValues(column.primitiveType, column.encoding, data.values, {
+    const valuesBuf = encodeValues(column.primitiveType, valueEncoding, data.values, {
         typeLength: column.typeLength,
-        bitWidth: column.typeLength
+        bitWidth: valueBitWidth ?? column.typeLength,
+        int96AsTimestamp
     });
     // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
     const compressedBuf = await Compression.deflate(column.compression, valuesBuf);
     /* encode repetition and definition levels */
-    let rLevelsBuf = Buffer.alloc(0);
+    let rLevelsBuf = new Uint8Array(0);
     if (column.rLevelMax > 0) {
         rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
             bitWidth: getBitWidth(column.rLevelMax),
             disableEnvelope: true
         });
     }
-    let dLevelsBuf = Buffer.alloc(0);
+    let dLevelsBuf = new Uint8Array(0);
     if (column.dLevelMax > 0) {
         dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
             bitWidth: getBitWidth(column.dLevelMax),
@@ -338,98 +428,677 @@ async function encodeDataPageV2(column, data, rowCount) {
             num_values: data.count,
             num_nulls: data.count - data.values.length,
             num_rows: rowCount,
-            encoding: Encoding[column.encoding],
+            encoding: Encoding[valueEncoding],
             definition_levels_byte_length: dLevelsBuf.length,
             repetition_levels_byte_length: rLevelsBuf.length,
-            is_compressed: column.compression !== 'UNCOMPRESSED'
+            is_compressed: column.compression !== 'UNCOMPRESSED',
+            statistics
         }),
         uncompressed_page_size: rLevelsBuf.length + dLevelsBuf.length + valuesBuf.length,
-        compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length
+        compressed_page_size: rLevelsBuf.length + dLevelsBuf.length + compressedBuf.length,
+        crc: writePageChecksums
+            ? crc32(concatUint8Arrays([rLevelsBuf, dLevelsBuf, compressedBuf]))
+            : undefined
     });
     /* concat page header, repetition and definition levels and values */
     const headerBuf = serializeThrift(header);
-    const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
+    const page = concatUint8Arrays([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
     return { header, headerSize: headerBuf.length, page };
 }
+/** Encodes one chunk-wide PLAIN dictionary page. */
+async function encodeDictionaryPage(column, dictionaryValues, writePageChecksums = false, int96AsTimestamp = false) {
+    const valuesBuffer = encodeValues(column.primitiveType, 'PLAIN', dictionaryValues, {
+        typeLength: column.typeLength,
+        int96AsTimestamp
+    });
+    const compressedBuffer = await Compression.deflate(column.compression, valuesBuffer);
+    const header = new PageHeader({
+        type: PageType.DICTIONARY_PAGE,
+        dictionary_page_header: new DictionaryPageHeader({
+            num_values: dictionaryValues.length,
+            encoding: Encoding.PLAIN
+        }),
+        uncompressed_page_size: valuesBuffer.length,
+        compressed_page_size: compressedBuffer.length,
+        crc: writePageChecksums ? crc32(compressedBuffer) : undefined
+    });
+    const headerBuffer = serializeThrift(header);
+    return {
+        header,
+        headerSize: headerBuffer.length,
+        page: concatUint8Arrays([headerBuffer, compressedBuffer])
+    };
+}
+/** Computes native Parquet geospatial statistics for a GEOMETRY or GEOGRAPHY column chunk. */
+function createGeospatialStatistics(column, values) {
+    const logicalType = column.logicalType?.type || column.originalType;
+    if (logicalType !== 'GEOMETRY' && logicalType !== 'GEOGRAPHY')
+        return undefined;
+    const geometryTypes = new Set();
+    let bbox;
+    // Vertex bounds are not conservative for GEOGRAPHY: non-linear edge algorithms can reach
+    // extrema between vertices. Until edge-aware geodesic bounds are available, omit the bbox while
+    // retaining the useful geometry-type statistics.
+    const canWriteBoundingBox = logicalType === 'GEOMETRY';
+    for (const value of values) {
+        if (!(value instanceof ArrayBuffer) && !ArrayBuffer.isView(value)) {
+            throw new Error(`${logicalType} columns must contain WKB binary values`);
+        }
+        const statistics = getWKBGeometryStatistics(value);
+        geometryTypes.add(statistics.geometryType);
+        if (canWriteBoundingBox) {
+            bbox = mergeGeometryBoundingBoxes(bbox, statistics.bbox);
+        }
+    }
+    return new GeospatialStatistics({
+        bbox: bbox ? new BoundingBox(bbox) : undefined,
+        geospatial_types: [...geometryTypes].sort((left, right) => left - right)
+    });
+}
+/** Merges finite per-dimension bounds without inventing absent Z or M dimensions. */
+function mergeGeometryBoundingBoxes(target, source) {
+    if (!source)
+        return target;
+    if (!target)
+        return { ...source };
+    target.xmin = Math.min(target.xmin, source.xmin);
+    target.xmax = Math.max(target.xmax, source.xmax);
+    target.ymin = Math.min(target.ymin, source.ymin);
+    target.ymax = Math.max(target.ymax, source.ymax);
+    mergeOptionalBounds(target, source, 'zmin', Math.min);
+    mergeOptionalBounds(target, source, 'zmax', Math.max);
+    mergeOptionalBounds(target, source, 'mmin', Math.min);
+    mergeOptionalBounds(target, source, 'mmax', Math.max);
+    return target;
+}
+/** Merges one optional bound while preserving a dimension absent from every value. */
+function mergeOptionalBounds(target, source, key, merge) {
+    const sourceValue = source[key];
+    if (sourceValue === undefined)
+        return;
+    target[key] = target[key] === undefined ? sourceValue : merge(target[key], sourceValue);
+}
 /**
  * Encode an array of values into a parquet column chunk
  */
-async function encodeColumnChunk(column, buffer, offset, opts) {
+async function encodeColumnChunk(column, buffer, offset, opts, rowGroupOrdinal, columnOrdinal) {
     const data = buffer.columnData[column.path.join()];
     const baseOffset = (opts.baseOffset || 0) + offset;
     /* encode data page(s) */
-    // const pages: Buffer[] = [];
-    let pageBuf;
+    const pageRecords = [];
+    const pageLocations = [];
     // tslint:disable-next-line:variable-name
     let total_uncompressed_size = 0;
     // tslint:disable-next-line:variable-name
     let total_compressed_size = 0;
-    {
+    const plannedPages = planColumnPages(column, data, buffer.rowCount, opts.pageSize || PARQUET_DEFAULT_PAGE_SIZE);
+    const dictionaryPolicy = opts.columnDictionaries?.[column.path[0]] ?? opts.dictionary ?? 'auto';
+    const dictionaryPlan = planDictionary(column, Array.from(data.values), dictionaryPolicy, opts.dictionaryPageSizeLimit ?? 1024 * 1024);
+    if (dictionaryPlan) {
+        const result = await encodeDictionaryPage(column, dictionaryPlan.values, opts.writePageChecksums, opts.int96AsTimestamp);
+        pageRecords.push({ page: result.page, headerSize: result.headerSize, dictionary: true });
+        total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
+        total_compressed_size += result.header.compressed_page_size + result.headerSize;
+    }
+    const valueEncoding = dictionaryPlan
+        ? column.encoding === 'PLAIN_DICTIONARY'
+            ? 'PLAIN_DICTIONARY'
+            : 'RLE_DICTIONARY'
+        : column.encoding === 'PLAIN_DICTIONARY' || column.encoding === 'RLE_DICTIONARY'
+            ? 'PLAIN'
+            : column.encoding;
+    let dictionaryIndexOffset = 0;
+    for (const plannedPage of plannedPages) {
+        const pageData = dictionaryPlan
+            ? {
+                ...plannedPage.data,
+                values: dictionaryPlan.indices.slice(dictionaryIndexOffset, dictionaryIndexOffset + plannedPage.data.values.length)
+            }
+            : plannedPage.data;
+        dictionaryIndexOffset += plannedPage.data.values.length;
         const result = opts.useDataPageV2
-            ? await encodeDataPageV2(column, data, buffer.rowCount)
-            : await encodeDataPage(column, data);
-        // pages.push(result.page);
-        pageBuf = result.page;
+            ? await encodeDataPageV2(column, pageData, plannedPage.rowCount, valueEncoding, dictionaryPlan?.bitWidth, opts.writePageChecksums, isStatisticsEnabled(opts.writeStatistics, column)
+                ? createColumnStatistics(column, plannedPage.data)
+                : undefined, opts.int96AsTimestamp)
+            : await encodeDataPage(column, pageData, valueEncoding, dictionaryPlan?.bitWidth, opts.writePageChecksums, isStatisticsEnabled(opts.writeStatistics, column)
+                ? createColumnStatistics(column, plannedPage.data)
+                : undefined, opts.int96AsTimestamp);
+        pageRecords.push({ page: result.page, headerSize: result.headerSize, dictionary: false });
         total_uncompressed_size += result.header.uncompressed_page_size + result.headerSize;
         total_compressed_size += result.header.compressed_page_size + result.headerSize;
     }
-    // const pagesBuf = Buffer.concat(pages);
+    const encryption = opts.encryption;
+    const encryptedColumn = Boolean(encryption && shouldEncryptColumn(encryption, column));
+    const columnKeyMetadata = encryption ? getColumnKeyMetadata(encryption, column) : undefined;
+    const pages = encryptedColumn
+        ? await encryptParquetColumnPages(pageRecords, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : pageRecords.map(record => record.page);
+    let pageOffset = 0;
+    let dataPageIndex = 0;
+    let firstRowIndex = 0;
+    for (const [pageIndex, page] of pages.entries()) {
+        if (!pageRecords[pageIndex].dictionary) {
+            pageLocations.push(new PageLocation({
+                offset: baseOffset + pageOffset,
+                compressed_page_size: page.length,
+                first_row_index: firstRowIndex
+            }));
+            firstRowIndex += plannedPages[dataPageIndex++].rowCount;
+        }
+        pageOffset += page.length;
+    }
+    total_compressed_size = pages.reduce((total, page) => total + page.length, 0);
+    const pagesBuf = concatUint8Arrays(pages);
     // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
+    const bloomFilterEnabled = opts.bloomFilter === true || opts.bloomFilter?.[column.path[0]] === true;
+    const bloomFilter = bloomFilterEnabled ? createColumnBloomFilter(column, data.values) : undefined;
+    const encodedBloomFilter = bloomFilter && encryptedColumn
+        ? await encryptParquetBloomFilter(bloomFilter, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : bloomFilter;
+    const pageIndexEnabled = opts.pageIndex === true || opts.pageIndex?.[column.path[0]] === true;
+    const pageIndexes = pageIndexEnabled
+        ? createColumnPageIndexes(column, plannedPages, pageLocations, opts.int96AsTimestamp)
+        : undefined;
+    const encodedPageIndexes = pageIndexes && encryptedColumn
+        ? await encryptParquetPageIndexes(pageIndexes, encryption, opts.encryptionFileUnique, opts.rowGroupOrdinal ?? 0, columnOrdinal, columnKeyMetadata)
+        : pageIndexes;
     /* prepare metadata header */
     const metadata = new ColumnMetaData({
         path_in_schema: column.path,
-        num_values: data.count,
-        data_page_offset: baseOffset,
+        num_values: int64(data.count),
+        data_page_offset: int64(baseOffset + (dictionaryPlan ? pages[0].length : 0)),
+        dictionary_page_offset: dictionaryPlan ? int64(baseOffset) : undefined,
         encodings: [],
-        total_uncompressed_size, //  : pagesBuf.length,
-        total_compressed_size,
+        encoding_stats: [
+            ...(dictionaryPlan
+                ? [
+                    new PageEncodingStats({
+                        page_type: PageType.DICTIONARY_PAGE,
+                        encoding: Encoding.PLAIN,
+                        count: 1
+                    })
+                ]
+                : []),
+            new PageEncodingStats({
+                page_type: opts.useDataPageV2 ? PageType.DATA_PAGE_V2 : PageType.DATA_PAGE,
+                encoding: Encoding[valueEncoding],
+                count: plannedPages.length
+            })
+        ],
+        total_uncompressed_size: int64(total_uncompressed_size), //  : pagesBuf.length,
+        total_compressed_size: int64(total_compressed_size),
         type: Type[column.primitiveType],
-        codec: CompressionCodec[column.compression]
+        codec: CompressionCodec[column.compression],
+        bloom_filter_offset: encodedBloomFilter ? int64(baseOffset + pagesBuf.length) : undefined,
+        bloom_filter_length: encodedBloomFilter?.length,
+        geospatial_statistics: createGeospatialStatistics(column, data.values),
+        size_statistics: opts.writeSizeStatistics ? createSizeStatistics(column, data) : undefined,
+        statistics: isStatisticsEnabled(opts.writeStatistics, column)
+            ? createColumnStatistics(column, data)
+            : undefined
     });
     /* list encodings */
     metadata.encodings.push(Encoding[PARQUET_RDLVL_ENCODING]);
-    metadata.encodings.push(Encoding[column.encoding]);
+    if (dictionaryPlan)
+        metadata.encodings.push(Encoding.PLAIN);
+    metadata.encodings.push(Encoding[valueEncoding]);
     /* concat metadata header and data pages */
-    const metadataOffset = baseOffset + pageBuf.length;
-    const body = Buffer.concat([pageBuf, serializeThrift(metadata)]);
-    return { body, metadata, metadataOffset };
+    const metadataEncoded = serializeThrift(metadata);
+    const metadataSection = encryptedColumn ? undefined : metadataEncoded;
+    const metadataOffset = baseOffset + pagesBuf.length + (encodedBloomFilter?.length || 0);
+    const offsetIndexOffset = encodedPageIndexes
+        ? metadataOffset + (metadataSection?.length || 0)
+        : undefined;
+    const columnIndexOffset = encodedPageIndexes?.columnIndex
+        ? metadataOffset + (metadataSection?.length || 0) + encodedPageIndexes.offsetIndex.length
+        : undefined;
+    const body = concatUint8Arrays([
+        pagesBuf,
+        ...(encodedBloomFilter ? [encodedBloomFilter] : []),
+        ...(metadataSection ? [metadataSection] : []),
+        ...(encodedPageIndexes
+            ? [
+                encodedPageIndexes.offsetIndex,
+                ...(encodedPageIndexes.columnIndex ? [encodedPageIndexes.columnIndex] : [])
+            ]
+            : [])
+    ]);
+    return {
+        body,
+        metadata,
+        metadataOffset,
+        offsetIndexOffset,
+        offsetIndexLength: encodedPageIndexes?.offsetIndex.length,
+        columnIndexOffset,
+        columnIndexLength: encodedPageIndexes?.columnIndex?.length,
+        ...(encryptedColumn
+            ? {
+                encryptedColumnMetadata: await encryptParquetModule(metadataEncoded, {
+                    algorithm: encryption.algorithm ?? 'AES_GCM_V1',
+                    aad: createParquetModuleAad(encryption.aadPrefix, opts.encryptionFileUnique, 'column-metadata', opts.rowGroupOrdinal ?? 0, columnOrdinal),
+                    keyMetadata: columnKeyMetadata ?? encryption.keyMetadata,
+                    keyRetriever: encryption.keyRetriever
+                }),
+                cryptoMetadata: new ColumnCryptoMetaData({
+                    ...(columnKeyMetadata
+                        ? {
+                            ENCRYPTION_WITH_COLUMN_KEY: new EncryptionWithColumnKey({
+                                path_in_schema: [...column.path],
+                                key_metadata: new Uint8Array(columnKeyMetadata)
+                            })
+                        }
+                        : { ENCRYPTION_WITH_FOOTER_KEY: new EncryptionWithFooterKey() })
+                })
+            }
+            : {})
+    };
+}
+/** Returns whether a writer column should use modular encryption. */
+function shouldEncryptColumn(options, column) {
+    return Boolean(options.columnKeyMetadata?.[column.path[0]] ||
+        options.encryptColumns === true ||
+        options.encryptColumns?.[column.path[0]] === true);
+}
+/** Returns a defensive copy of the key metadata assigned to one top-level column. */
+function getColumnKeyMetadata(options, column) {
+    const keyMetadata = options.columnKeyMetadata?.[column.path[0]];
+    return keyMetadata ? new Uint8Array(keyMetadata) : undefined;
+}
+/** Adds the four-byte module length required by encrypted page and Bloom-filter modules. */
+function prefixEncryptedModule(module) {
+    const prefixed = new Uint8Array(module.byteLength + 4);
+    writeUInt32LE(prefixed, module.byteLength, 0);
+    prefixed.set(module, 4);
+    return prefixed;
+}
+/** Encrypts one column's dictionary and data pages using the Parquet module AAD rules. */
+async function encryptParquetColumnPages(pageRecords, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const encryptedPages = [];
+    let dataPageOrdinal = 0;
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    for (const pageRecord of pageRecords) {
+        const moduleType = pageRecord.dictionary ? 'dictionary-page' : 'data-page';
+        const headerModuleType = pageRecord.dictionary ? 'dictionary-page-header' : 'data-page-header';
+        const header = await encryptParquetModule(pageRecord.page.subarray(0, pageRecord.headerSize), {
+            algorithm,
+            aad: createParquetModuleAad(options.aadPrefix, fileUnique, headerModuleType, rowGroupOrdinal, columnOrdinal, pageRecord.dictionary ? undefined : dataPageOrdinal),
+            keyMetadata: keyMetadata ?? options.keyMetadata,
+            keyRetriever: options.keyRetriever
+        });
+        const body = await encryptParquetModule(pageRecord.page.subarray(pageRecord.headerSize), {
+            algorithm,
+            aad: createParquetModuleAad(options.aadPrefix, fileUnique, moduleType, rowGroupOrdinal, columnOrdinal, pageRecord.dictionary ? undefined : dataPageOrdinal),
+            keyMetadata: keyMetadata ?? options.keyMetadata,
+            keyRetriever: options.keyRetriever,
+            page: true
+        });
+        encryptedPages.push(concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(body)]));
+        if (!pageRecord.dictionary)
+            dataPageOrdinal++;
+    }
+    return encryptedPages;
+}
+/** Encrypts a split-block Bloom filter header and bitset as independent Parquet modules. */
+async function encryptParquetBloomFilter(bloomFilter, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const parsed = decodeParquetSplitBlockBloomFilter(bloomFilter);
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const keyOptions = {
+        algorithm,
+        keyMetadata: keyMetadata ?? options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    };
+    const header = await encryptParquetModule(bloomFilter.subarray(0, parsed.headerByteLength), {
+        ...keyOptions,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'bloom-filter-header', rowGroupOrdinal, columnOrdinal)
+    });
+    const bitset = await encryptParquetModule(parsed.bitset, {
+        ...keyOptions,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'bloom-filter-bitset', rowGroupOrdinal, columnOrdinal)
+    });
+    return concatUint8Arrays([prefixEncryptedModule(header), prefixEncryptedModule(bitset)]);
+}
+/** Encrypts column and offset indexes independently while preserving their footer offsets. */
+async function encryptParquetPageIndexes(pageIndexes, options, fileUnique, rowGroupOrdinal, columnOrdinal, keyMetadata) {
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const encryptIndex = (bytes, module) => encryptParquetModule(bytes, {
+        algorithm,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, module, rowGroupOrdinal, columnOrdinal),
+        keyMetadata: keyMetadata ?? options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    });
+    return {
+        offsetIndex: await encryptIndex(pageIndexes.offsetIndex, 'offset-index'),
+        columnIndex: pageIndexes.columnIndex
+            ? await encryptIndex(pageIndexes.columnIndex, 'column-index')
+            : undefined
+    };
+}
+/** Builds page indexes for primitive leaves, including nested and repeated leaves. */
+function createColumnPageIndexes(column, plannedPages, pageLocations, int96AsTimestamp = false) {
+    if (!column.primitiveType ||
+        !isPageIndexPhysicalType(column.primitiveType) ||
+        plannedPages.length !== pageLocations.length ||
+        plannedPages.length === 0) {
+        return undefined;
+    }
+    const offsetIndex = serializeThrift(new OffsetIndex({ page_locations: pageLocations }));
+    if (!supportsStatisticsSortOrder(column)) {
+        return { offsetIndex };
+    }
+    const nullPages = [];
+    const minValues = [];
+    const maxValues = [];
+    // A repeated leaf's definition-level count is not a row-level null count.
+    // Omit null_counts for those leaves so readers do not interpret entry counts
+    // as null counts while still benefiting from min/max page pruning.
+    const nullCounts = column.rLevelMax === 0 ? [] : undefined;
+    for (const plannedPage of plannedPages) {
+        const values = plannedPage.data.values;
+        const nullCount = plannedPage.data.count - values.length;
+        nullCounts?.push(nullCount);
+        if (values.length === 0) {
+            nullPages.push(true);
+            minValues.push(new Uint8Array(0));
+            maxValues.push(new Uint8Array(0));
+            continue;
+        }
+        const bounds = getPageBounds(values, column.primitiveType, column.typeLength, int96AsTimestamp);
+        if (!bounds)
+            return undefined;
+        nullPages.push(false);
+        minValues.push(bounds.min);
+        maxValues.push(bounds.max);
+    }
+    return {
+        offsetIndex,
+        columnIndex: serializeThrift(new ColumnIndex({
+            null_pages: nullPages,
+            min_values: minValues,
+            max_values: maxValues,
+            boundary_order: BoundaryOrder.UNORDERED,
+            null_counts: nullCounts
+        }))
+    };
+}
+function isPageIndexPhysicalType(physicalType) {
+    return (physicalType === 'BOOLEAN' ||
+        physicalType === 'INT32' ||
+        physicalType === 'INT64' ||
+        physicalType === 'INT96' ||
+        physicalType === 'FLOAT' ||
+        physicalType === 'DOUBLE' ||
+        physicalType === 'BYTE_ARRAY' ||
+        physicalType === 'FIXED_LEN_BYTE_ARRAY');
+}
+function getPageBounds(values, physicalType, typeLength, int96AsTimestamp = false) {
+    if (physicalType === 'INT96' && !int96AsTimestamp)
+        return undefined;
+    const normalizedValues = Array.from(values, value => normalizePageIndexValue(value, physicalType));
+    if (normalizedValues.some(value => value === undefined))
+        return undefined;
+    const comparableValues = normalizedValues;
+    let min = comparableValues[0];
+    let max = comparableValues[0];
+    for (const value of comparableValues.slice(1)) {
+        if (comparePageIndexValues(value, min, physicalType) < 0)
+            min = value;
+        if (comparePageIndexValues(value, max, physicalType) > 0)
+            max = value;
+    }
+    try {
+        return {
+            min: encodePageIndexValue(min, physicalType, typeLength, int96AsTimestamp),
+            max: encodePageIndexValue(max, physicalType, typeLength, int96AsTimestamp)
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function encodePageIndexValue(value, physicalType, typeLength, int96AsTimestamp) {
+    if (physicalType === 'INT96') {
+        if (!int96AsTimestamp) {
+            throw new Error('INT96 page indexes require timestamp mode');
+        }
+        return PARQUET_CODECS.PLAIN.encodeValues('INT96', [value], { int96AsTimestamp });
+    }
+    return encodeParquetBloomFilterValue(value, physicalType, typeLength);
+}
+function normalizePageIndexValue(value, physicalType) {
+    if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
+        if (typeof value === 'string')
+            return new TextEncoder().encode(value);
+        if (value instanceof ArrayBuffer)
+            return new Uint8Array(value);
+        if (ArrayBuffer.isView(value)) {
+            return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
+        }
+        return undefined;
+    }
+    if (typeof value === 'boolean' ||
+        typeof value === 'number' ||
+        typeof value === 'bigint' ||
+        typeof value === 'string') {
+        return value;
+    }
+    return undefined;
+}
+function comparePageIndexValues(left, right, physicalType) {
+    if (physicalType === 'BYTE_ARRAY' || physicalType === 'FIXED_LEN_BYTE_ARRAY') {
+        const leftBytes = typeof left === 'string' ? new TextEncoder().encode(left) : left;
+        const rightBytes = typeof right === 'string' ? new TextEncoder().encode(right) : right;
+        if (!(leftBytes instanceof Uint8Array) || !(rightBytes instanceof Uint8Array))
+            return 0;
+        const length = Math.min(leftBytes.length, rightBytes.length);
+        for (let index = 0; index < length; index++) {
+            if (leftBytes[index] !== rightBytes[index]) {
+                return leftBytes[index] < rightBytes[index] ? -1 : 1;
+            }
+        }
+        return leftBytes.length - rightBytes.length;
+    }
+    if (typeof left === 'boolean' && typeof right === 'boolean')
+        return Number(left) - Number(right);
+    if (physicalType !== 'INT64' && physicalType !== 'INT96') {
+        const leftNumber = Number(left);
+        const rightNumber = Number(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+    try {
+        const leftNumber = BigInt(left);
+        const rightNumber = BigInt(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+    catch {
+        const leftNumber = Number(left);
+        const rightNumber = Number(right);
+        return leftNumber < rightNumber ? -1 : leftNumber > rightNumber ? 1 : 0;
+    }
+}
+/** Builds a Bloom filter for writer-supported physical scalar columns. */
+function createColumnBloomFilter(column, values) {
+    const physicalType = column.primitiveType;
+    if (physicalType !== 'BOOLEAN' &&
+        physicalType !== 'INT32' &&
+        physicalType !== 'INT64' &&
+        physicalType !== 'FLOAT' &&
+        physicalType !== 'DOUBLE' &&
+        physicalType !== 'BYTE_ARRAY' &&
+        physicalType !== 'FIXED_LEN_BYTE_ARRAY') {
+        return undefined;
+    }
+    const bloomValues = values.map(value => {
+        if (value instanceof ArrayBuffer)
+            return new Uint8Array(value);
+        if (ArrayBuffer.isView(value)) {
+            return new Uint8Array(value.buffer, value.byteOffset, value.byteLength).slice();
+        }
+        return value;
+    });
+    return encodeParquetSplitBlockBloomFilter(bloomValues, physicalType, column.typeLength);
+}
+/** Builds optional size statistics from one shredded column chunk. */
+function createSizeStatistics(column, data) {
+    const repetitionLevelHistogram = createLevelHistogram(data.rlevels, column.rLevelMax);
+    const definitionLevelHistogram = createLevelHistogram(data.dlevels, column.dLevelMax);
+    let unencodedByteArrayDataBytes = 0;
+    if (column.primitiveType === 'BYTE_ARRAY') {
+        for (const value of data.values) {
+            if (typeof value === 'string') {
+                unencodedByteArrayDataBytes += encodeUtf8(value).byteLength;
+            }
+            else if (value instanceof ArrayBuffer || ArrayBuffer.isView(value)) {
+                unencodedByteArrayDataBytes += value.byteLength;
+            }
+        }
+    }
+    return new SizeStatistics({
+        unencoded_byte_array_data_bytes: column.primitiveType === 'BYTE_ARRAY' ? unencodedByteArrayDataBytes : undefined,
+        repetition_level_histogram: repetitionLevelHistogram,
+        definition_level_histogram: definitionLevelHistogram
+    });
+}
+/** Returns whether standard column statistics are enabled for one top-level field. */
+function isStatisticsEnabled(option, column) {
+    return option === true || option?.[column.path[0]] === true;
+}
+/** Builds conservative standard min/max/null-count statistics for one column chunk. */
+function createColumnStatistics(column, data) {
+    if (!column.primitiveType ||
+        !isPageIndexPhysicalType(column.primitiveType) ||
+        !supportsStatisticsSortOrder(column)) {
+        return undefined;
+    }
+    const statistics = new Statistics({
+        null_count: column.rLevelMax === 0 ? data.count - data.values.length : undefined,
+        is_min_value_exact: true,
+        is_max_value_exact: true
+    });
+    if (data.values.length === 0)
+        return statistics;
+    const bounds = getPageBounds(data.values, column.primitiveType, column.typeLength);
+    if (!bounds)
+        return undefined;
+    statistics.min_value = bounds.min;
+    statistics.max_value = bounds.max;
+    return statistics;
+}
+/** Returns whether the physical representation preserves the logical sort order. */
+function supportsStatisticsSortOrder(column) {
+    const logicalType = column.logicalType?.type || column.originalType;
+    if (logicalType === 'FLOAT16')
+        return false;
+    if (logicalType === 'DECIMAL' || logicalType?.startsWith('DECIMAL_')) {
+        return column.primitiveType !== 'BYTE_ARRAY' && column.primitiveType !== 'FIXED_LEN_BYTE_ARRAY';
+    }
+    if (logicalType === 'INTEGER' || logicalType?.startsWith('UINT_')) {
+        return column.logicalType?.isSigned !== false && !logicalType.startsWith('UINT_');
+    }
+    return true;
+}
+/** Counts the occurrences of each definition or repetition level. */
+function createLevelHistogram(levels, maximumLevel) {
+    const histogram = new Array(maximumLevel + 1).fill(0);
+    for (const level of levels) {
+        if (level >= 0 && level <= maximumLevel)
+            histogram[level]++;
+    }
+    return histogram;
+}
+/** Converts writer-facing sort keys into Parquet leaf-column indexes. */
+function createSortingColumns(schema, sortingColumns) {
+    if (!sortingColumns?.length)
+        return undefined;
+    const leafFields = schema.fieldList.filter(field => !field.isNested);
+    const usedColumnIndexes = new Set();
+    return sortingColumns.map(sortKey => {
+        const columnIndex = leafFields.findIndex(field => (field.path.length === 1 && field.name === sortKey.column) ||
+            (field.path.length > 1 && field.path.join('.') === sortKey.column));
+        if (columnIndex < 0) {
+            throw new Error(`Unknown Parquet sorting column ${sortKey.column}`);
+        }
+        if (usedColumnIndexes.has(columnIndex)) {
+            throw new Error(`Duplicate Parquet sorting column ${sortKey.column}`);
+        }
+        usedColumnIndexes.add(columnIndex);
+        return new SortingColumn({
+            column_idx: columnIndex,
+            descending: Boolean(sortKey.descending),
+            nulls_first: Boolean(sortKey.nullsFirst)
+        });
+    });
 }
 /**
  * Encode a list of column values into a parquet row group
  */
 async function encodeRowGroup(schema, data, opts) {
     const metadata = new RowGroup({
-        num_rows: data.rowCount,
+        num_rows: int64(data.rowCount),
         columns: [],
-        total_byte_size: 0
+        total_byte_size: int64(0),
+        sorting_columns: createSortingColumns(schema, opts.sortingColumns)
     });
-    let body = Buffer.alloc(0);
+    let body = new Uint8Array(0);
     for (const field of schema.fieldList) {
         if (field.isNested) {
             continue; // eslint-disable-line no-continue
         }
-        const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
+        const cchunkData = await encodeColumnChunk(field, data, body.length, opts, opts.rowGroupOrdinal ?? 0, metadata.columns.length);
         const cchunk = new ColumnChunk({
-            file_offset: cchunkData.metadataOffset,
-            meta_data: cchunkData.metadata
+            file_offset: int64(cchunkData.metadataOffset),
+            meta_data: cchunkData.encryptedColumnMetadata ? undefined : cchunkData.metadata,
+            offset_index_offset: cchunkData.offsetIndexOffset === undefined
+                ? undefined
+                : int64(cchunkData.offsetIndexOffset),
+            offset_index_length: cchunkData.offsetIndexLength,
+            column_index_offset: cchunkData.columnIndexOffset === undefined
+                ? undefined
+                : int64(cchunkData.columnIndexOffset),
+            column_index_length: cchunkData.columnIndexLength,
+            crypto_metadata: cchunkData.cryptoMetadata,
+            encrypted_column_metadata: cchunkData.encryptedColumnMetadata
         });
         metadata.columns.push(cchunk);
-        metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
-        body = Buffer.concat([body, cchunkData.body]);
+        metadata.total_byte_size = int64(Number(metadata.total_byte_size) + cchunkData.body.length);
+        body = concatUint8Arrays([body, cchunkData.body]);
     }
     return { body, metadata };
 }
 /**
  * Encode a parquet file metadata footer
  */
-function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
+function encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, footerSignature) {
+    const signatureAlgorithm = footerSignature?.algorithm ?? 'AES_GCM_V1';
+    const signatureFileUnique = footerSignature?.fileUnique
+        ? new Uint8Array(footerSignature.fileUnique)
+        : undefined;
     const metadata = new FileMetaData({
         version: PARQUET_VERSION,
         created_by: 'parquets',
-        num_rows: rowCount,
+        num_rows: int64(rowCount),
         row_groups: rowGroups,
         schema: [],
-        key_value_metadata: []
+        key_value_metadata: [],
+        encryption_algorithm: footerSignature
+            ? new EncryptionAlgorithm({
+                ...(signatureAlgorithm === 'AES_GCM_CTR_V1'
+                    ? {
+                        AES_GCM_CTR_V1: {
+                            aad_prefix: footerSignature.aadPrefix,
+                            aad_file_unique: signatureFileUnique
+                        }
+                    }
+                    : {
+                        AES_GCM_V1: {
+                            aad_prefix: footerSignature.aadPrefix,
+                            aad_file_unique: signatureFileUnique
+                        }
+                    })
+            })
+            : undefined,
+        footer_signing_key_metadata: footerSignature
+            ? new Uint8Array(footerSignature.keyMetadata)
+            : undefined
     });
     for (const key in userMetadata) {
         const kv = new KeyValue({
@@ -458,16 +1127,211 @@ function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
             schemaElem.type = Type[field.primitiveType];
         }
         if (field.originalType) {
-            schemaElem.converted_type = ConvertedType[field.originalType];
+            const convertedType = ConvertedType[field.originalType];
+            if (typeof convertedType === 'number') {
+                schemaElem.converted_type = convertedType;
+            }
         }
         schemaElem.type_length = field.typeLength;
+        schemaElem.precision = field.precision ?? field.presision;
+        schemaElem.scale = field.scale;
+        schemaElem.field_id = field.fieldId;
+        const logicalType = getFieldLogicalType(field);
+        if (logicalType) {
+            schemaElem.logicalType = encodeParquetLogicalType(logicalType);
+        }
         metadata.schema.push(schemaElem);
     }
-    const metadataEncoded = serializeThrift(metadata);
-    const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
-    metadataEncoded.copy(footerEncoded);
-    footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
-    footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
+    return serializeThrift(metadata);
+}
+/** Encodes an unsigned plaintext footer with its standard Parquet trailer. */
+function encodeFooter(schema, rowCount, rowGroups, userMetadata) {
+    const metadataEncoded = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata);
+    const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
+    footerEncoded.set(metadataEncoded);
+    writeUInt32LE(footerEncoded, metadataEncoded.length, metadataEncoded.length);
+    footerEncoded.set(PARQUET_MAGIC_BYTES, metadataEncoded.length + 4);
     return footerEncoded;
 }
+/** Encodes a plaintext footer with its trailing 28-byte authenticated signature. */
+async function encodeSignedFooter(schema, rowCount, rowGroups, userMetadata, options) {
+    const metadata = encodeFooterMetadata(schema, rowCount, rowGroups, userMetadata, options);
+    const signature = await createParquetFooterSignature(metadata, options);
+    const footerEncoded = new Uint8Array(metadata.length + signature.length + 8);
+    footerEncoded.set(metadata);
+    footerEncoded.set(signature, metadata.length);
+    writeUInt32LE(footerEncoded, metadata.length + signature.length, metadata.length + signature.length);
+    footerEncoded.set(PARQUET_MAGIC_BYTES, metadata.length + signature.length + 4);
+    return footerEncoded;
+}
+/** Wraps a serialized footer in Parquet modular-encryption metadata and AES-GCM. */
+async function encodeEncryptedFooter(footer, options) {
+    const algorithm = options.algorithm ?? 'AES_GCM_V1';
+    const fileUnique = options.fileUnique ? new Uint8Array(options.fileUnique) : createFileUnique();
+    if (fileUnique.byteLength !== 8) {
+        throw new Error(`Parquet encrypted footer file_unique must be 8 bytes`);
+    }
+    const algorithmParameters = {
+        aad_prefix: options.aadPrefix && new Uint8Array(options.aadPrefix),
+        aad_file_unique: fileUnique,
+        supply_aad_prefix: undefined
+    };
+    const encryptionAlgorithm = algorithm === 'AES_GCM_CTR_V1'
+        ? new EncryptionAlgorithm({ AES_GCM_CTR_V1: algorithmParameters })
+        : new EncryptionAlgorithm({ AES_GCM_V1: algorithmParameters });
+    const cryptoMetadata = serializeThrift(new FileCryptoMetaData({
+        encryption_algorithm: encryptionAlgorithm,
+        key_metadata: options.keyMetadata && new Uint8Array(options.keyMetadata)
+    }));
+    const footerMetadata = footer.subarray(0, footer.byteLength - 8);
+    const encryptedFooter = await encryptParquetModule(footerMetadata, {
+        algorithm,
+        aad: createParquetModuleAad(options.aadPrefix, fileUnique, 'footer'),
+        keyMetadata: options.keyMetadata,
+        keyRetriever: options.keyRetriever
+    });
+    const metadataSize = cryptoMetadata.byteLength + encryptedFooter.byteLength;
+    const trailer = new Uint8Array(8);
+    writeUInt32LE(trailer, metadataSize, 0);
+    trailer.set(encodeUtf8(PARQUET_MAGIC_ENCRYPTED), 4);
+    return concatUint8Arrays([cryptoMetadata, encryptedFooter, trailer]);
+}
+/** Generates the eight-byte file identifier required by Parquet module AAD. */
+function createFileUnique() {
+    const fileUnique = new Uint8Array(8);
+    const cryptoProvider = globalThis.crypto;
+    if (!cryptoProvider?.getRandomValues) {
+        throw new Error('Parquet encrypted footer requires Web Crypto random values');
+    }
+    cryptoProvider.getRandomValues(fileUnique);
+    return fileUnique;
+}
+/** Returns an explicit or inferred modern logical annotation for a writer field. */
+function getFieldLogicalType(field) {
+    if (field.logicalType)
+        return field.logicalType;
+    const originalType = field.originalType;
+    if (!originalType)
+        return undefined;
+    if (originalType === 'UTF8')
+        return { type: 'STRING' };
+    if (originalType === 'ENUM')
+        return { type: 'ENUM' };
+    if (originalType.startsWith('DECIMAL_')) {
+        return {
+            type: 'DECIMAL',
+            precision: field.precision ?? field.presision,
+            scale: field.scale
+        };
+    }
+    if (originalType === 'DATE')
+        return { type: 'DATE' };
+    if (originalType.startsWith('TIME_')) {
+        return {
+            type: 'TIME',
+            unit: originalType.slice(5),
+            isAdjustedToUTC: true
+        };
+    }
+    if (originalType.startsWith('TIMESTAMP_')) {
+        return {
+            type: 'TIMESTAMP',
+            unit: originalType.slice(10),
+            isAdjustedToUTC: true
+        };
+    }
+    if (originalType.startsWith('UINT_') || originalType.startsWith('INT_')) {
+        return {
+            type: 'INTEGER',
+            bitWidth: Number(originalType.slice(originalType.indexOf('_') + 1)),
+            isSigned: originalType.startsWith('INT_')
+        };
+    }
+    if (originalType === 'JSON' || originalType === 'BSON')
+        return { type: originalType };
+    if (originalType === 'UUID' ||
+        originalType === 'FLOAT16' ||
+        originalType === 'UNKNOWN' ||
+        originalType === 'VARIANT' ||
+        originalType === 'GEOMETRY' ||
+        originalType === 'GEOGRAPHY') {
+        return { type: originalType };
+    }
+    return undefined;
+}
+/** Converts the internal logical annotation into the Parquet 2.13 Thrift union. */
+function encodeParquetLogicalType(logicalType) {
+    switch (logicalType.type) {
+        case 'STRING':
+            return LogicalType.fromSTRING(new StringType());
+        case 'MAP':
+            return LogicalType.fromMAP(new MapType());
+        case 'LIST':
+            return LogicalType.fromLIST(new ListType());
+        case 'ENUM':
+            return LogicalType.fromENUM(new EnumType());
+        case 'DECIMAL':
+            if (logicalType.precision === undefined || logicalType.scale === undefined) {
+                throw new Error('Parquet DECIMAL requires precision and scale');
+            }
+            return LogicalType.fromDECIMAL(new DecimalType({ precision: logicalType.precision, scale: logicalType.scale }));
+        case 'DATE':
+            return LogicalType.fromDATE(new DateType());
+        case 'TIME':
+            return LogicalType.fromTIME(new TimeType({
+                isAdjustedToUTC: logicalType.isAdjustedToUTC ?? false,
+                unit: encodeParquetTimeUnit(logicalType.unit)
+            }));
+        case 'TIMESTAMP':
+            return LogicalType.fromTIMESTAMP(new TimestampType({
+                isAdjustedToUTC: logicalType.isAdjustedToUTC ?? false,
+                unit: encodeParquetTimeUnit(logicalType.unit)
+            }));
+        case 'INTEGER':
+            if (logicalType.bitWidth === undefined || logicalType.isSigned === undefined) {
+                throw new Error('Parquet INTEGER requires bitWidth and isSigned');
+            }
+            return LogicalType.fromINTEGER(new IntType({ bitWidth: logicalType.bitWidth, isSigned: logicalType.isSigned }));
+        case 'UNKNOWN':
+            return LogicalType.fromUNKNOWN(new NullType());
+        case 'JSON':
+            return LogicalType.fromJSON(new JsonType());
+        case 'BSON':
+            return LogicalType.fromBSON(new BsonType());
+        case 'UUID':
+            return LogicalType.fromUUID(new UUIDType());
+        case 'FLOAT16':
+            return LogicalType.fromFLOAT16(new Float16Type());
+        case 'VARIANT':
+            return LogicalType.fromVARIANT(new VariantType({ specification_version: logicalType.specificationVersion }));
+        case 'GEOMETRY':
+            return LogicalType.fromGEOMETRY(new GeometryType({ crs: logicalType.crs }));
+        case 'GEOGRAPHY':
+            return LogicalType.fromGEOGRAPHY(new GeographyType({
+                crs: logicalType.crs,
+                algorithm: logicalType.algorithm === undefined
+                    ? undefined
+                    : EdgeInterpolationAlgorithm[logicalType.algorithm]
+            }));
+        default:
+            throw new Error(`Unsupported Parquet logical type ${logicalType.type}`);
+    }
+}
+/** Encodes one Parquet logical time unit. */
+function encodeParquetTimeUnit(unit) {
+    switch (unit) {
+        case 'MILLIS':
+            return TimeUnit.fromMILLIS(new MilliSeconds());
+        case 'MICROS':
+            return TimeUnit.fromMICROS(new MicroSeconds());
+        case 'NANOS':
+            return TimeUnit.fromNANOS(new NanoSeconds());
+        default:
+            throw new Error('Parquet TIME and TIMESTAMP require a unit');
+    }
+}
+/** Wrap a writer metadata number in the thrift int64 compatibility object. */
+function int64(value) {
+    return new CompactInt64(BigInt(value));
+}
 //# sourceMappingURL=parquet-encoder.js.map