@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/src/parquetjs/schema/declare.ts

@@ -4,10 +4,20 @@
 // Copyright (c) 2017 ironSource Ltd.
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
-import Int64 from 'node-int64';
+import {CompactInt64 as Int64} from '../utils/uint8-array-compact-protocol';
 import type {PageHeader} from '../parquet-thrift';
+import type {ParquetValueBuffer} from '../codecs/declare';
 
-export type ParquetCodec = 'PLAIN' | 'RLE' | 'PLAIN_DICTIONARY';
+export type ParquetCodec =
+  | 'PLAIN'
+  | 'RLE'
+  | 'BIT_PACKED'
+  | 'PLAIN_DICTIONARY'
+  | 'RLE_DICTIONARY'
+  | 'DELTA_BINARY_PACKED'
+  | 'DELTA_LENGTH_BYTE_ARRAY'
+  | 'DELTA_BYTE_ARRAY'
+  | 'BYTE_STREAM_SPLIT';
 export type ParquetCompression =
   | 'UNCOMPRESSED'
   | 'GZIP'
@@ -52,8 +62,10 @@ export type OriginalType =
   | 'DATE' // 6
   | 'TIME_MILLIS' // 7
   | 'TIME_MICROS' // 8
+  | 'TIME_NANOS'
   | 'TIMESTAMP_MILLIS' // 9
   | 'TIMESTAMP_MICROS' // 10
+  | 'TIMESTAMP_NANOS'
   | 'UINT_8' // 11
   | 'UINT_16' // 12
   | 'UINT_32' // 13
@@ -62,11 +74,65 @@ export type OriginalType =
   | 'INT_16' // 16
   | 'INT_32' // 17
   | 'INT_64' // 18
+  | 'ENUM'
+  | 'UUID'
+  | 'FLOAT16'
+  | 'UNKNOWN'
+  | 'VARIANT'
+  | 'GEOMETRY'
+  | 'GEOGRAPHY'
   | 'JSON' // 19
   | 'BSON' // 20
   | 'INTERVAL'; // 21
 
-export type ParquetDictionary = string[];
+/** Units carried by Parquet TIME and TIMESTAMP logical type annotations. */
+export type ParquetTimeUnit = 'MILLIS' | 'MICROS' | 'NANOS';
+
+/** Logical type names defined by the Parquet 2.13 format. */
+export type ParquetLogicalTypeName =
+  | 'STRING'
+  | 'MAP'
+  | 'LIST'
+  | 'ENUM'
+  | 'DECIMAL'
+  | 'DATE'
+  | 'TIME'
+  | 'TIMESTAMP'
+  | 'INTEGER'
+  | 'UNKNOWN'
+  | 'JSON'
+  | 'BSON'
+  | 'UUID'
+  | 'FLOAT16'
+  | 'VARIANT'
+  | 'GEOMETRY'
+  | 'GEOGRAPHY';
+
+/** Serializable representation of one Parquet logical type annotation. */
+export interface ParquetLogicalType {
+  /** Logical type discriminator. */
+  type: ParquetLogicalTypeName;
+  /** Integer width for INTEGER annotations. */
+  bitWidth?: 8 | 16 | 32 | 64;
+  /** Whether an INTEGER annotation is signed. */
+  isSigned?: boolean;
+  /** Time unit for TIME and TIMESTAMP annotations. */
+  unit?: ParquetTimeUnit;
+  /** Whether a TIME or TIMESTAMP value represents a UTC-normalized instant. */
+  isAdjustedToUTC?: boolean;
+  /** Decimal precision. */
+  precision?: number;
+  /** Decimal scale. */
+  scale?: number;
+  /** Variant specification version. */
+  specificationVersion?: number;
+  /** Coordinate reference system for geospatial logical types. */
+  crs?: string;
+  /** Edge interpolation algorithm for GEOGRAPHY values. */
+  algorithm?: string;
+}
+
+export type ParquetDictionary = any[];
 
 export interface SchemaDefinition {
   [string: string]: FieldDefinition;
@@ -74,9 +140,18 @@ export interface SchemaDefinition {
 
 export interface FieldDefinition {
   type?: ParquetType;
+  /** Physical type declared by the file, retained independently from its logical type. */
+  physicalType?: PrimitiveType;
   typeLength?: number;
+  /** @deprecated Use `precision`. */
   presision?: number;
+  /** Decimal precision. */
+  precision?: number;
   scale?: number;
+  /** Modern Parquet logical type annotation. */
+  logicalType?: ParquetLogicalType;
+  /** Stable field identifier declared by the Parquet schema. */
+  fieldId?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
   optional?: boolean;
@@ -90,9 +165,16 @@ export interface ParquetField {
   key: string;
   primitiveType?: PrimitiveType;
   originalType?: OriginalType;
+  /** Modern Parquet logical type annotation. */
+  logicalType?: ParquetLogicalType;
+  /** Stable field identifier declared by the Parquet schema. */
+  fieldId?: number;
   repetitionType: RepetitionType;
   typeLength?: number;
+  /** @deprecated Use `precision`. */
   presision?: number;
+  /** Decimal precision. */
+  precision?: number;
   scale?: number;
   encoding?: ParquetCodec;
   compression?: ParquetCompression;
@@ -112,15 +194,36 @@ export interface ParquetReaderContext {
   column: ParquetField;
   numValues?: Int64;
   dictionary?: ParquetDictionary;
+  /** Reader-scoped codec selected once and reused for independently compressed pages. */
+  decompressPage?: (value: Uint8Array, size: number) => Promise<Uint8Array>;
   /** If true, binary values are not converted to strings */
   preserveBinary?: boolean;
+  /** Retain byte arrays as views into decoded page buffers for direct materialization. */
+  retainByteArrayViews?: boolean;
+  /** Decode primitive values into typed column buffers when their physical type permits it. */
+  useTypedValueBuffers?: boolean;
+  /** Decode repetition and definition levels into compact unsigned typed arrays. */
+  useTypedLevelBuffers?: boolean;
+  /** Decode eligible PLAIN BYTE_ARRAY columns into Arrow-compatible contiguous buffers. */
+  useArrowByteArrayBuffers?: boolean;
+  /** Column metadata guarantees every data page supports direct Arrow byte-array decoding. */
+  hasOnlyArrowByteArrayDataPages?: boolean;
+  /** Verify a page-header CRC when one is present. */
+  verifyPageChecksums?: boolean;
+  /** Decode legacy INT96 physical values as epoch nanoseconds. */
+  int96AsTimestamp?: boolean;
 }
 
+/** Mutable storage for decoded Parquet repetition and definition levels. */
+export type ParquetLevelBuffer = number[] | Uint8Array | Uint16Array | Uint32Array;
+
 export interface ParquetPageData {
-  dlevels: number[];
-  rlevels: number[];
+  dlevels: ParquetLevelBuffer;
+  rlevels: ParquetLevelBuffer;
   /** Actual column chunks */
-  values: any[]; // ArrayLike<any>;
+  values: ParquetValueBuffer;
+  /** Number of values written directly into the column destination, if one was supplied. */
+  directValuesWritten?: number;
   count: number;
   dictionary?: ParquetDictionary;
   /** The "raw" page header from the file */
@@ -147,9 +250,18 @@ export class ParquetRowGroup {
 
 /** Holds the data for one column chunk */
 export interface ParquetColumnChunk {
-  dlevels: number[];
-  rlevels: number[];
-  values: any[];
+  dlevels: ParquetLevelBuffer;
+  rlevels: ParquetLevelBuffer;
+  values: ParquetValueBuffer;
+  /** Number of omitted physical values represented by definition levels when known. */
+  nullCount?: number;
+  /** Compact Arrow-compatible physical BYTE_ARRAY data produced by the Arrow reader path. */
+  byteArrayData?: {
+    /** Contiguous non-null physical value bytes. */
+    data: Uint8Array;
+    /** Offsets into `data`, with one entry beyond the final physical value. */
+    valueOffsets: Int32Array;
+  };
   count: number;
   pageHeaders: PageHeader[];
 }

package/src/parquetjs/schema/schema.ts

@@ -5,6 +5,8 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license
 
 import {PARQUET_CODECS} from '../codecs/index';
+import {isByteStreamSplitType} from '../codecs/byte-stream-split';
+import {isDeltaEncodingType} from '../codecs/delta';
 import {PARQUET_COMPRESSION_METHODS} from '../compression';
 import {
   FieldDefinition,
@@ -15,7 +17,8 @@ import {
   RepetitionType,
   SchemaDefinition
 } from './declare';
-import {materializeRows, shredBuffer, shredRecord} from './shred';
+import type {ArrayType} from '@loaders.gl/schema';
+import {materializeColumns, materializeRows, shredBuffer, shredRecord} from './shred';
 import {PARQUET_LOGICAL_TYPES} from './types';
 
 /**
@@ -40,9 +43,19 @@ export class ParquetSchema {
    */
   findField(path: string | string[]): ParquetField {
     if (typeof path === 'string') {
+      // Flat schemas dominate analytical Parquet. Avoid split/array allocation for every Arrow
+      // field lookup while retaining the existing traversal for genuinely nested paths.
+      if (!path.includes(',')) {
+        return this.fields[path];
+      }
       // tslint:disable-next-line:no-parameter-reassignment
       path = path.split(',');
     } else {
+      // Column metadata already stores paths as arrays; the one-segment case needs no defensive
+      // clone or shift and is exercised once per selected flat column.
+      if (path.length === 1) {
+        return this.fields[path[0]];
+      }
       // tslint:disable-next-line:no-parameter-reassignment
       path = path.slice(0); // clone array
     }
@@ -82,6 +95,11 @@ export class ParquetSchema {
     return materializeRows(this, rowGroup);
   }
 
+  /** Materializes one decoded row group as top-level column arrays. */
+  materializeColumns(rowGroup: ParquetRowGroup): Record<string, ArrayType> {
+    return materializeColumns(this, rowGroup);
+  }
+
   compress(type: ParquetCompression): this {
     setCompress(this.schema, type);
     setCompress(this.fields, type);
@@ -140,6 +158,8 @@ function buildFields(
         name,
         path: cpath,
         key: cpath.join(),
+        logicalType: opts.logicalType,
+        fieldId: opts.fieldId,
         repetitionType,
         rLevelMax,
         dLevelMax,
@@ -159,6 +179,13 @@ function buildFields(
     if (!(opts.encoding in PARQUET_CODECS)) {
       throw new Error(`unsupported parquet encoding: ${opts.encoding}`);
     }
+    const primitiveType = opts.physicalType || typeDef.primitiveType;
+    if (opts.encoding === 'BYTE_STREAM_SPLIT' && !isByteStreamSplitType(primitiveType)) {
+      throw new Error(`BYTE_STREAM_SPLIT does not support ${primitiveType}`);
+    }
+    if (opts.encoding.startsWith('DELTA_') && !isDeltaEncodingType(opts.encoding, primitiveType)) {
+      throw new Error(`${opts.encoding} does not support ${primitiveType}`);
+    }
 
     opts.compression = opts.compression || 'UNCOMPRESSED';
     if (!(opts.compression in PARQUET_COMPRESSION_METHODS)) {
@@ -169,15 +196,18 @@ function buildFields(
     const cpath = path.concat([name]);
     fieldList[name] = {
       name,
-      primitiveType: typeDef.primitiveType,
+      primitiveType,
       originalType: typeDef.originalType,
+      logicalType: opts.logicalType,
+      fieldId: opts.fieldId,
       path: cpath,
       key: cpath.join(),
       repetitionType,
       encoding: opts.encoding,
       compression: opts.compression,
       typeLength: opts.typeLength || typeDef.typeLength,
-      presision: opts.presision,
+      presision: opts.precision ?? opts.presision,
+      precision: opts.precision ?? opts.presision,
       scale: opts.scale,
       rLevelMax,
       dLevelMax