@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/src/parquetjs/parser/parquet-reader.ts

@@ -17,19 +17,44 @@ import {
   ParquetCompression,
   ParquetColumnChunk,
   PrimitiveType,
-  ParquetReaderContext
+  ParquetReaderContext,
+  type ParquetLevelBuffer
 } from '../schema/declare';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
+import {decodeString, readUInt32LE, toUint8Array} from '../utils/binary-utils';
+import {CompactInt64} from '../utils/uint8-array-compact-protocol';
+import type {
+  ParquetDataPageLocation,
+  ParquetPageLocations,
+  ParquetRowRange
+} from '../../lib/parquet-page-index';
+
+/** Bounds concurrent range requests when a row group contains unusually many selected columns. */
+const MAXIMUM_CONCURRENT_COLUMN_READS = 16;
 
 export type ParquetReaderProps = {
+  /** Maximum dictionary-page read size. */
   defaultDictionarySize?: number;
+  /** Preserve BYTE_ARRAY values instead of decoding them as strings. */
   preserveBinary?: boolean;
+  /** Retain byte arrays as views into decoded page buffers for direct materialization. */
+  retainByteArrayViews?: boolean;
+  /** Decode supported primitive columns into typed buffers instead of boxed JavaScript arrays. */
+  useTypedValueBuffers?: boolean;
+  /** Decode repetition and definition levels into compact unsigned typed arrays. */
+  useTypedLevelBuffers?: boolean;
+  /** Abort signal forwarded to every underlying random-access read. */
+  signal?: AbortSignal;
 };
 
 /** Properties for initializing a ParquetRowGroupReader */
 export type ParquetIterationProps = {
   /** Filter allowing some columns to be dropped */
   columnList?: string[] | string[][];
+  /** Zero-based row-group indexes to read, in output order. */
+  rowGroups?: number[];
+  /** Abort signal forwarded to row-group and column-chunk reads. */
+  signal?: AbortSignal;
 };
 
 /**
@@ -39,15 +64,21 @@ export type ParquetIterationProps = {
  * rows from a parquet file use the ParquetReader instead
  */
 export class ParquetReader {
-  static defaultProps: Required<ParquetReaderProps> = {
+  static defaultProps: Required<Omit<ParquetReaderProps, 'signal'>> & {signal?: AbortSignal} = {
     // max ArrayBuffer size in js is 2Gb
     defaultDictionarySize: 2147483648,
-    preserveBinary: false
+    preserveBinary: false,
+    retainByteArrayViews: false,
+    useTypedValueBuffers: false,
+    useTypedLevelBuffers: false,
+    signal: undefined
   };
 
-  props: Required<ParquetReaderProps>;
+  props: Required<Omit<ParquetReaderProps, 'signal'>> & {signal?: AbortSignal};
   file: ReadableFile;
   metadata: Promise<FileMetaData> | null = null;
+  /** Parsed Parquet schema shared by metadata, iteration, and materialization paths. */
+  private schema: Promise<ParquetSchema> | null = null;
 
   constructor(file: ReadableFile, props?: ParquetReaderProps) {
     this.file = file;
@@ -82,36 +113,41 @@ export class ParquetReader {
   /** Iterate over the raw row groups */
   async *rowGroupIterator(props?: ParquetIterationProps) {
     // Ensure strings are nested in arrays
-    const columnList: string[][] = (props?.columnList || []).map((x) =>
-      Array.isArray(x) ? x : [x]
-    );
+    const columnList: string[][] = (props?.columnList || []).map(x => (Array.isArray(x) ? x : [x]));
 
-    const metadata = await this.getFileMetadata();
-    const schema = await this.getSchema();
+    const metadata = await this.getFileMetadata(props?.signal);
+    const schema = await this.getSchema(props?.signal);
 
     const rowGroupCount = metadata?.row_groups.length || 0;
+    const rowGroupIndices =
+      props?.rowGroups || Array.from({length: rowGroupCount}, (_, index) => index);
 
-    for (let rowGroupIndex = 0; rowGroupIndex < rowGroupCount; rowGroupIndex++) {
+    for (const rowGroupIndex of rowGroupIndices) {
+      if (!Number.isInteger(rowGroupIndex) || rowGroupIndex < 0 || rowGroupIndex >= rowGroupCount) {
+        throw new Error(`Invalid Parquet row-group index ${rowGroupIndex}`);
+      }
       const rowGroup = await this.readRowGroup(
         schema,
         metadata.row_groups[rowGroupIndex],
-        columnList
+        columnList,
+        props?.signal
       );
       yield rowGroup;
     }
   }
 
-  async getRowCount(): Promise<number> {
-    const metadata = await this.getFileMetadata();
+  async getRowCount(signal?: AbortSignal): Promise<number> {
+    const metadata = await this.getFileMetadata(signal);
     return Number(metadata.num_rows);
   }
 
-  async getSchema(): Promise<ParquetSchema> {
-    const metadata = await this.getFileMetadata();
-    const root = metadata.schema[0];
-    const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);
-    const schema = new ParquetSchema(schemaDefinition);
-    return schema;
+  async getSchema(signal?: AbortSignal): Promise<ParquetSchema> {
+    this.schema ||= this.getFileMetadata(signal).then(metadata => {
+      const root = metadata.schema[0];
+      const {schema: schemaDefinition} = decodeSchema(metadata.schema, 1, root.num_children!);
+      return new ParquetSchema(schemaDefinition);
+    });
+    return await this.schema;
   }
 
   /**
@@ -127,10 +163,10 @@ export class ParquetReader {
     return md;
   }
 
-  async getFileMetadata(): Promise<FileMetaData> {
+  async getFileMetadata(signal?: AbortSignal): Promise<FileMetaData> {
     if (!this.metadata) {
-      await this.readHeader();
-      this.metadata = this.readFooter();
+      await this.readHeader(signal);
+      this.metadata = this.readFooter(signal);
     }
     return this.metadata;
   }
@@ -138,10 +174,9 @@ export class ParquetReader {
   // LOW LEVEL METHODS
 
   /** Metadata is stored in the footer */
-  async readHeader(): Promise<void> {
-    const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);
-    const buffer = Buffer.from(arrayBuffer);
-    const magic = buffer.toString();
+  async readHeader(signal?: AbortSignal): Promise<void> {
+    const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length, signal ?? this.props.signal);
+    const magic = decodeString(toUint8Array(arrayBuffer));
     switch (magic) {
       case PARQUET_MAGIC:
         break;
@@ -153,24 +188,32 @@ export class ParquetReader {
   }
 
   /** Metadata is stored in the footer */
-  async readFooter(): Promise<FileMetaData> {
+  async readFooter(signal?: AbortSignal): Promise<FileMetaData> {
     const trailerLen = PARQUET_MAGIC.length + 4;
-    const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);
-    const trailerBuf = Buffer.from(arrayBuffer);
+    const arrayBuffer = await this.file.read(
+      this.file.size - trailerLen,
+      trailerLen,
+      signal ?? this.props.signal
+    );
+    const trailer = toUint8Array(arrayBuffer);
 
-    const magic = trailerBuf.slice(4).toString();
+    const magic = decodeString(trailer, 4);
     if (magic !== PARQUET_MAGIC) {
       throw new Error(`Not a valid parquet file (magic="${magic})`);
     }
 
-    const metadataSize = trailerBuf.readUInt32LE(0);
+    const metadataSize = readUInt32LE(trailer, 0);
     const metadataOffset = this.file.size - metadataSize - trailerLen;
     if (metadataOffset < PARQUET_MAGIC.length) {
       throw new Error(`Invalid metadata size ${metadataOffset}`);
     }
 
-    const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);
-    const metadataBuf = Buffer.from(arrayBuffer2);
+    const arrayBuffer2 = await this.file.read(
+      metadataOffset,
+      metadataSize,
+      signal ?? this.props.signal
+    );
+    const metadataBuf = toUint8Array(arrayBuffer2);
     // let metadata = new parquet_thrift.FileMetaData();
     // parquet_util.decodeThrift(metadata, metadataBuf);
 
@@ -182,27 +225,83 @@ export class ParquetReader {
   async readRowGroup(
     schema: ParquetSchema,
     rowGroup: RowGroup,
-    columnList: string[][]
+    columnList: string[][],
+    signal?: AbortSignal
   ): Promise<ParquetRowGroup> {
-    const buffer: ParquetRowGroup = {
+    const selectedColumnChunks = rowGroup.columns.filter(columnChunk => {
+      const columnKey = columnChunk.meta_data?.path_in_schema;
+      return columnList.length === 0 || fieldIndexOf(columnList, columnKey!) >= 0;
+    });
+    const columnEntries: Array<readonly [string, ParquetColumnChunk]> = [];
+    for (
+      let batchStart = 0;
+      batchStart < selectedColumnChunks.length;
+      batchStart += MAXIMUM_CONCURRENT_COLUMN_READS
+    ) {
+      const columnBatch = selectedColumnChunks.slice(
+        batchStart,
+        batchStart + MAXIMUM_CONCURRENT_COLUMN_READS
+      );
+      columnEntries.push(
+        ...(await Promise.all(
+          columnBatch.map(async columnChunk => {
+            const columnKey = columnChunk.meta_data!.path_in_schema.join();
+            const columnData = await this.readColumnChunk(schema, columnChunk, signal);
+            return [columnKey, columnData] as const;
+          })
+        ))
+      );
+    }
+    return {
       rowCount: Number(rowGroup.num_rows),
-      columnData: {}
+      columnData: Object.fromEntries(columnEntries)
+    };
+  }
+
+  /** Reads independently materializable flat columns for one logical row range using page indexes. */
+  async readRowGroupRange(
+    schema: ParquetSchema,
+    rowGroup: RowGroup,
+    columnList: string[][],
+    rowRange: ParquetRowRange,
+    pageLocations: ParquetPageLocations,
+    signal?: AbortSignal
+  ): Promise<ParquetRowGroup> {
+    const selectedColumnChunks = rowGroup.columns.filter(columnChunk => {
+      const columnKey = columnChunk.meta_data?.path_in_schema;
+      return columnList.length === 0 || fieldIndexOf(columnList, columnKey!) >= 0;
+    });
+    const columnEntries = await Promise.all(
+      selectedColumnChunks.map(async columnChunk => {
+        const columnKey = columnChunk.meta_data!.path_in_schema.join();
+        const pages = pageLocations[columnChunk.meta_data!.path_in_schema.join('.')];
+        if (!pages) {
+          throw new Error(`Parquet offset index missing for ${columnKey}`);
+        }
+        const columnData = await this.readColumnChunkRange(
+          schema,
+          columnChunk,
+          rowRange,
+          pages,
+          signal
+        );
+        return [columnKey, columnData] as const;
+      })
+    );
+    return {
+      rowCount: rowRange.end - rowRange.start,
+      columnData: Object.fromEntries(columnEntries)
     };
-    for (const colChunk of rowGroup.columns) {
-      const colMetadata = colChunk.meta_data;
-      const colKey = colMetadata?.path_in_schema;
-      if (columnList.length > 0 && fieldIndexOf(columnList, colKey!) < 0) {
-        continue; // eslint-disable-line no-continue
-      }
-      buffer.columnData[colKey!.join()] = await this.readColumnChunk(schema, colChunk);
-    }
-    return buffer;
   }
 
   /**
    * Each row group contains column chunks for all the columns.
    */
-  async readColumnChunk(schema: ParquetSchema, colChunk: ColumnChunk): Promise<ParquetColumnChunk> {
+  async readColumnChunk(
+    schema: ParquetSchema,
+    colChunk: ColumnChunk,
+    signal?: AbortSignal
+  ): Promise<ParquetColumnChunk> {
     if (colChunk.file_path !== undefined && colChunk.file_path !== null) {
       throw new Error('external references are not supported');
     }
@@ -220,14 +319,21 @@ export class ParquetReader {
     ) as any;
 
     const pagesOffset = Number(colChunk.meta_data?.data_page_offset!);
-    let pagesSize = Number(colChunk.meta_data?.total_compressed_size!);
-
-    if (!colChunk.file_path) {
-      pagesSize = Math.min(
-        this.file.size - pagesOffset,
-        Number(colChunk.meta_data?.total_compressed_size)
-      );
-    }
+    const dictionaryPageOffset = colChunk.meta_data?.dictionary_page_offset;
+    const validDictionaryPageOffset =
+      dictionaryPageOffset !== undefined && Number(dictionaryPageOffset) > 0
+        ? Number(dictionaryPageOffset)
+        : undefined;
+    const chunkOffset = Math.min(pagesOffset, validDictionaryPageOffset ?? pagesOffset);
+    const chunkEnd = chunkOffset + Number(colChunk.meta_data?.total_compressed_size!);
+    const chunkSize = Math.min(this.file.size - chunkOffset, Math.max(0, chunkEnd - chunkOffset));
+    const arrayBuffer = await this.file.read(chunkOffset, chunkSize, signal ?? this.props.signal);
+    const chunkBuffer = toUint8Array(arrayBuffer);
+    const pagesRelativeOffset = pagesOffset - chunkOffset;
+    const pagesSize = Math.min(
+      chunkBuffer.length - pagesRelativeOffset,
+      Math.max(0, chunkEnd - pagesOffset)
+    );
 
     const context: ParquetReaderContext = {
       type,
@@ -238,25 +344,98 @@ export class ParquetReader {
       numValues: colChunk.meta_data?.num_values,
       dictionary: [],
       // Options - TBD is this the right place for these?
-      preserveBinary: this.props.preserveBinary
+      preserveBinary: this.props.preserveBinary,
+      retainByteArrayViews: this.props.retainByteArrayViews,
+      useTypedValueBuffers: this.props.useTypedValueBuffers,
+      useTypedLevelBuffers: this.props.useTypedLevelBuffers
     };
 
-    let dictionary;
+    let dictionary: any[] | undefined;
 
-    const dictionaryPageOffset = colChunk?.meta_data?.dictionary_page_offset;
-
-    if (dictionaryPageOffset) {
-      const dictionaryOffset = Number(dictionaryPageOffset);
-      // Getting dictionary from column chunk to iterate all over indexes to get dataPage values.
-      dictionary = await this.getDictionary(dictionaryOffset, context, pagesOffset);
+    if (validDictionaryPageOffset !== undefined) {
+      const dictionaryRelativeOffset = validDictionaryPageOffset - chunkOffset;
+      const dictionarySize = Math.min(
+        Math.max(0, pagesOffset - validDictionaryPageOffset),
+        chunkBuffer.length - dictionaryRelativeOffset,
+        this.props.defaultDictionarySize
+      );
+      const dictionaryBuffer = chunkBuffer.subarray(
+        dictionaryRelativeOffset,
+        dictionaryRelativeOffset + dictionarySize
+      );
+      dictionary = await decodeDictionaryBuffer(dictionaryBuffer, context);
     }
 
     dictionary = context.dictionary?.length ? context.dictionary : dictionary;
-    const arrayBuffer = await this.file.read(pagesOffset, pagesSize);
-    const pagesBuf = Buffer.from(arrayBuffer);
+    const pagesBuf = chunkBuffer.subarray(pagesRelativeOffset, pagesRelativeOffset + pagesSize);
     return await decodeDataPages(pagesBuf, {...context, dictionary});
   }
 
+  /** Reads and decodes only the contiguous data pages overlapping one flat-column row range. */
+  async readColumnChunkRange(
+    schema: ParquetSchema,
+    columnChunk: ColumnChunk,
+    rowRange: ParquetRowRange,
+    pages: readonly ParquetDataPageLocation[],
+    signal?: AbortSignal
+  ): Promise<ParquetColumnChunk> {
+    if (columnChunk.file_path !== undefined && columnChunk.file_path !== null) {
+      throw new Error('external references are not supported');
+    }
+    const columnMetadata = columnChunk.meta_data!;
+    const field = schema.findField(columnMetadata.path_in_schema);
+    if (field.path.length !== 1 || field.repetitionType === 'REPEATED' || field.rLevelMax !== 0) {
+      throw new Error('Selective Parquet page reads currently require flat primitive columns');
+    }
+    const type: PrimitiveType = getThriftEnum(Type, columnMetadata.type) as any;
+    if (type !== field.primitiveType) {
+      throw new Error(`chunk type not matching schema: ${type}`);
+    }
+    const compression: ParquetCompression = getThriftEnum(
+      CompressionCodec,
+      columnMetadata.codec
+    ) as any;
+    const overlappingPages = pages.filter(
+      page => page.endRowIndex > rowRange.start && page.firstRowIndex < rowRange.end
+    );
+    if (!overlappingPages.length) {
+      throw new Error('Parquet page plan does not overlap the requested row range');
+    }
+    const firstPage = overlappingPages[0];
+    const lastPage = overlappingPages[overlappingPages.length - 1];
+    const context: ParquetReaderContext = {
+      type,
+      rLevelMax: field.rLevelMax,
+      dLevelMax: field.dLevelMax,
+      compression,
+      column: field,
+      numValues: new CompactInt64(lastPage.endRowIndex - firstPage.firstRowIndex),
+      dictionary: [],
+      preserveBinary: this.props.preserveBinary,
+      retainByteArrayViews: this.props.retainByteArrayViews,
+      useTypedValueBuffers: this.props.useTypedValueBuffers
+    };
+
+    let dictionary: any[] | undefined;
+    const dictionaryPageOffset = Number(columnMetadata.dictionary_page_offset);
+    if (Number.isSafeInteger(dictionaryPageOffset) && dictionaryPageOffset > 0) {
+      const dictionaryLength = Math.max(0, pages[0].offset - dictionaryPageOffset);
+      const dictionaryBuffer = toUint8Array(
+        await this.file.read(dictionaryPageOffset, dictionaryLength, signal ?? this.props.signal)
+      );
+      dictionary = await decodeDictionaryBuffer(dictionaryBuffer, context);
+    }
+
+    const dataLength = lastPage.offset + lastPage.compressedByteLength - firstPage.offset;
+    const dataBuffer = toUint8Array(
+      await this.file.read(firstPage.offset, dataLength, signal ?? this.props.signal)
+    );
+    const decoded = await decodeDataPages(dataBuffer, {...context, dictionary});
+    const relativeStart = rowRange.start - firstPage.firstRowIndex;
+    const relativeEnd = relativeStart + rowRange.end - rowRange.start;
+    return sliceFlatColumnChunk(decoded, field.dLevelMax, relativeStart, relativeEnd);
+  }
+
   /**
    * Getting dictionary for allows to flatten values by indices.
    * @param dictionaryPageOffset
@@ -267,8 +446,9 @@ export class ParquetReader {
   async getDictionary(
     dictionaryPageOffset: number,
     context: ParquetReaderContext,
-    pagesOffset: number
-  ): Promise<string[]> {
+    pagesOffset: number,
+    signal?: AbortSignal
+  ): Promise<any[]> {
     if (dictionaryPageOffset === 0) {
       // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
       // pagesBuf = await this.read(pagesOffset, dictionarySize);
@@ -281,15 +461,61 @@ export class ParquetReader {
     }
 
     const dictionarySize = Math.min(
+      Math.max(0, pagesOffset - dictionaryPageOffset),
       this.file.size - dictionaryPageOffset,
       this.props.defaultDictionarySize
     );
-    const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);
-    const pagesBuf = Buffer.from(arrayBuffer);
+    const arrayBuffer = await this.file.read(
+      dictionaryPageOffset,
+      dictionarySize,
+      signal ?? this.props.signal
+    );
+    const pagesBuf = toUint8Array(arrayBuffer);
+    return await decodeDictionaryBuffer(pagesBuf, context);
+  }
+}
+
+/** Decodes one dictionary page from an already-read column-chunk range. */
+async function decodeDictionaryBuffer(
+  dictionaryBuffer: Uint8Array,
+  context: ParquetReaderContext
+): Promise<any[]> {
+  const cursor = {buffer: dictionaryBuffer, offset: 0, size: dictionaryBuffer.length};
+  const decodedPage = await decodePage(cursor, context);
+  return decodedPage.dictionary!;
+}
 
-    const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
-    const decodedPage = await decodePage(cursor, context);
+/** Slices one decoded flat column chunk while preserving optional-value alignment. */
+function sliceFlatColumnChunk(
+  columnChunk: ParquetColumnChunk,
+  definitionLevelMaximum: number,
+  start: number,
+  end: number
+): ParquetColumnChunk {
+  const valueStart = countDefinedValues(columnChunk.dlevels, definitionLevelMaximum, 0, start);
+  const valueEnd =
+    valueStart + countDefinedValues(columnChunk.dlevels, definitionLevelMaximum, start, end);
+  return {
+    rlevels: columnChunk.rlevels.slice(start, end),
+    dlevels: columnChunk.dlevels.slice(start, end),
+    values: columnChunk.values.slice(valueStart, valueEnd) as typeof columnChunk.values,
+    count: end - start,
+    pageHeaders: columnChunk.pageHeaders
+  };
+}
 
-    return decodedPage.dictionary!;
+/** Counts defined primitive values represented by one flat level interval. */
+function countDefinedValues(
+  definitionLevels: ParquetLevelBuffer,
+  definitionLevelMaximum: number,
+  start: number,
+  end: number
+): number {
+  let count = 0;
+  for (let index = start; index < Math.min(end, definitionLevels.length); index++) {
+    if (definitionLevels[index] === definitionLevelMaximum) {
+      count++;
+    }
   }
+  return count;
 }

package/src/parquetjs/schema/declare.ts

@@ -4,10 +4,19 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
-import Int64 from 'node-int64';
+import {CompactInt64 as Int64} from '../utils/uint8-array-compact-protocol';
 import type {PageHeader} from '../parquet-thrift';
+import type {ParquetValueBuffer} from '../codecs/declare';
 
-export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
+export type ParquetCodec =
+  | 'PLAIN'
+  | 'RLE'
+  | 'PLAIN_DICTIONARY'
+  | 'RLE_DICTIONARY'
+  | 'DELTA_BINARY_PACKED'
+  | 'DELTA_LENGTH_BYTE_ARRAY'
+  | 'DELTA_BYTE_ARRAY'
+  | 'BYTE_STREAM_SPLIT';
 export type ParquetCompression =
   | 'UNCOMPRESSED'
   | 'GZIP'
@@ -52,8 +61,10 @@ export type OriginalType =
   | 'DATE' // 6
   | 'TIME_MILLIS' // 7
   | 'TIME_MICROS' // 8
+  | 'TIME_NANOS'
   | 'TIMESTAMP_MILLIS' // 9
   | 'TIMESTAMP_MICROS' // 10
+  | 'TIMESTAMP_NANOS'
   | 'UINT_8' // 11
   | 'UINT_16' // 12
   | 'UINT_32' // 13
@@ -62,11 +73,65 @@ export type OriginalType =
   | 'INT_16' // 16
   | 'INT_32' // 17
   | 'INT_64' // 18
+  | 'ENUM'
+  | 'UUID'
+  | 'FLOAT16'
+  | 'UNKNOWN'
+  | 'VARIANT'
+  | 'GEOMETRY'
+  | 'GEOGRAPHY'
   | 'JSON' // 19
   | 'BSON' // 20
   | 'INTERVAL'; // 21
 
-export type ParquetDictionary = string[];
+/** Units carried by Parquet TIME and TIMESTAMP logical type annotations. */
+export type ParquetTimeUnit = 'MILLIS' | 'MICROS' | 'NANOS';
+
+/** Logical type names defined by the Parquet 2.13 format. */
+export type ParquetLogicalTypeName =
+  | 'STRING'
+  | 'MAP'
+  | 'LIST'
+  | 'ENUM'
+  | 'DECIMAL'
+  | 'DATE'
+  | 'TIME'
+  | 'TIMESTAMP'
+  | 'INTEGER'
+  | 'UNKNOWN'
+  | 'JSON'
+  | 'BSON'
+  | 'UUID'
+  | 'FLOAT16'
+  | 'VARIANT'
+  | 'GEOMETRY'
+  | 'GEOGRAPHY';
+
+/** Serializable representation of one Parquet logical type annotation. */
+export interface ParquetLogicalType {
+  /** Logical type discriminator. */
+  type: ParquetLogicalTypeName;
+  /** Integer width for INTEGER annotations. */
+  bitWidth?: 8 | 16 | 32 | 64;
+  /** Whether an INTEGER annotation is signed. */
+  isSigned?: boolean;
+  /** Time unit for TIME and TIMESTAMP annotations. */
+  unit?: ParquetTimeUnit;
+  /** Whether a TIME or TIMESTAMP value represents a UTC-normalized instant. */
+  isAdjustedToUTC?: boolean;
+  /** Decimal precision. */
+  precision?: number;
+  /** Decimal scale. */
+  scale?: number;
+  /** Variant specification version. */
+  specificationVersion?: number;
+  /** Coordinate reference system for geospatial logical types. */
+  crs?: string;
+  /** Edge interpolation algorithm for GEOGRAPHY values. */
+  algorithm?: string;
+}
+
+export type ParquetDictionary = any[];
 
 export interface SchemaDefinition {
   [string: string]: FieldDefinition;
@@ -74,9 +139,18 @@ export interface SchemaDefinition {
 
 export interface FieldDefinition {
   type?: ParquetType;
+  /** Physical type declared by the file, retained independently from its logical type. */
+  physicalType?: PrimitiveType;
   typeLength?: number;
+  /** @deprecated Use `precision`. */
   presision?: number;
+  /** Decimal precision. */
+  precision?: number;
   scale?: number;
+  /** Modern Parquet logical type annotation. */
+  logicalType?: ParquetLogicalType;
+  /** Stable field identifier declared by the Parquet schema. */
+  fieldId?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
   optional?: boolean;
@@ -90,9 +164,16 @@ export interface ParquetField {
   key: string;
   primitiveType?: PrimitiveType;
   originalType?: OriginalType;
+  /** Modern Parquet logical type annotation. */
+  logicalType?: ParquetLogicalType;
+  /** Stable field identifier declared by the Parquet schema. */
+  fieldId?: number;
   repetitionType: RepetitionType;
   typeLength?: number;
+  /** @deprecated Use `precision`. */
   presision?: number;
+  /** Decimal precision. */
+  precision?: number;
   scale?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
@@ -114,13 +195,24 @@ export interface ParquetReaderContext {
   dictionary?: ParquetDictionary;
   /** If true, binary values are not converted to strings */
   preserveBinary?: boolean;
+  /** Retain byte arrays as views into decoded page buffers for direct materialization. */
+  retainByteArrayViews?: boolean;
+  /** Decode primitive values into typed column buffers when their physical type permits it. */
+  useTypedValueBuffers?: boolean;
+  /** Decode repetition and definition levels into compact unsigned typed arrays. */
+  useTypedLevelBuffers?: boolean;
 }
 
+/** Mutable storage for decoded Parquet repetition and definition levels. */
+export type ParquetLevelBuffer = number[] | Uint8Array | Uint16Array | Uint32Array;
+
 export interface ParquetPageData {
   dlevels: number[];
   rlevels: number[];
   /** Actual column chunks */
-  values: any[]; // ArrayLike<any>;
+  values: ParquetValueBuffer;
+  /** Number of values written directly into the column destination, if one was supplied. */
+  directValuesWritten?: number;
   count: number;
   dictionary?: ParquetDictionary;
   /** The "raw" page header from the file */
@@ -147,9 +239,9 @@ export class ParquetRowGroup {
 
 /** Holds the data for one column chunk */
 export interface ParquetColumnChunk {
-  dlevels: number[];
-  rlevels: number[];
-  values: any[];
+  dlevels: ParquetLevelBuffer;
+  rlevels: ParquetLevelBuffer;
+  values: ParquetValueBuffer;
   count: number;
   pageHeaders: PageHeader[];
 }