@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

package/src/parquetjs/schema/declare.ts

@@ -11,6 +11,7 @@ import type {ParquetValueBuffer} from '../codecs/declare';
 export type ParquetCodec =
   | 'PLAIN'
   | 'RLE'
+  | 'BIT_PACKED'
   | 'PLAIN_DICTIONARY'
   | 'RLE_DICTIONARY'
   | 'DELTA_BINARY_PACKED'
@@ -193,6 +194,8 @@ export interface ParquetReaderContext {
   column: ParquetField;
   numValues?: Int64;
   dictionary?: ParquetDictionary;
+  /** Reader-scoped codec selected once and reused for independently compressed pages. */
+  decompressPage?: (value: Uint8Array, size: number) => Promise<Uint8Array>;
   /** If true, binary values are not converted to strings */
   preserveBinary?: boolean;
   /** Retain byte arrays as views into decoded page buffers for direct materialization. */
@@ -201,14 +204,22 @@ export interface ParquetReaderContext {
   useTypedValueBuffers?: boolean;
   /** Decode repetition and definition levels into compact unsigned typed arrays. */
   useTypedLevelBuffers?: boolean;
+  /** Decode eligible PLAIN BYTE_ARRAY columns into Arrow-compatible contiguous buffers. */
+  useArrowByteArrayBuffers?: boolean;
+  /** Column metadata guarantees every data page supports direct Arrow byte-array decoding. */
+  hasOnlyArrowByteArrayDataPages?: boolean;
+  /** Verify a page-header CRC when one is present. */
+  verifyPageChecksums?: boolean;
+  /** Decode legacy INT96 physical values as epoch nanoseconds. */
+  int96AsTimestamp?: boolean;
 }
 
 /** Mutable storage for decoded Parquet repetition and definition levels. */
 export type ParquetLevelBuffer = number[] | Uint8Array | Uint16Array | Uint32Array;
 
 export interface ParquetPageData {
-  dlevels: number[];
-  rlevels: number[];
+  dlevels: ParquetLevelBuffer;
+  rlevels: ParquetLevelBuffer;
   /** Actual column chunks */
   values: ParquetValueBuffer;
   /** Number of values written directly into the column destination, if one was supplied. */
@@ -242,6 +253,15 @@ export interface ParquetColumnChunk {
   dlevels: ParquetLevelBuffer;
   rlevels: ParquetLevelBuffer;
   values: ParquetValueBuffer;
+  /** Number of omitted physical values represented by definition levels when known. */
+  nullCount?: number;
+  /** Compact Arrow-compatible physical BYTE_ARRAY data produced by the Arrow reader path. */
+  byteArrayData?: {
+    /** Contiguous non-null physical value bytes. */
+    data: Uint8Array;
+    /** Offsets into `data`, with one entry beyond the final physical value. */
+    valueOffsets: Int32Array;
+  };
   count: number;
   pageHeaders: PageHeader[];
 }

package/src/parquetjs/schema/schema.ts

@@ -43,9 +43,19 @@ export class ParquetSchema {
    */
   findField(path: string | string[]): ParquetField {
     if (typeof path === 'string') {
+      // Flat schemas dominate analytical Parquet. Avoid split/array allocation for every Arrow
+      // field lookup while retaining the existing traversal for genuinely nested paths.
+      if (!path.includes(',')) {
+        return this.fields[path];
+      }
       // tslint:disable-next-line:no-parameter-reassignment
       path = path.split(',');
     } else {
+      // Column metadata already stores paths as arrays; the one-segment case needs no defensive
+      // clone or shift and is exercised once per selected flat column.
+      if (path.length === 1) {
+        return this.fields[path[0]];
+      }
       // tslint:disable-next-line:no-parameter-reassignment
       path = path.slice(0); // clone array
     }

package/src/parquetjs/schema/shred.ts

@@ -8,6 +8,7 @@ import {ArrayType} from '@loaders.gl/schema';
 import {ParquetRowGroup, ParquetColumnChunk, ParquetField, ParquetRow} from './declare';
 import {ParquetSchema} from './schema';
 import * as Types from './types';
+import {decodeVariant} from './variant';
 
 export {ParquetRowGroup};
 
@@ -100,10 +101,33 @@ function shredRecordFields(
       record[field.name] !== undefined &&
       record[field.name] !== null
     ) {
-      if (record[field.name].constructor === Array) {
-        values = record[field.name];
+      const fieldValue = record[field.name];
+      if (field.logicalType?.type === 'LIST') {
+        // Normalize the high-level Arrow/list value to Parquet's standard
+        // three-level LIST representation before descending into the wrapper.
+        const listValues = Array.isArray(fieldValue) ? fieldValue : [];
+        if (field.fields?.list) {
+          values.push({list: listValues.map(element => ({element}))});
+        } else {
+          // Accept the legacy two-level LIST layout where the repeated field
+          // itself owns the element leaf.
+          values = listValues;
+        }
+      } else if (field.logicalType?.type === 'MAP') {
+        // Normalize Map/object/entry-array values to the standard MAP_KEY_VALUE
+        // wrapper expected by the shredding algorithm.
+        const mapEntries = normalizeMapEntries(fieldValue);
+        if (field.fields?.key_value) {
+          values.push({key_value: mapEntries});
+        } else {
+          // Accept legacy map layouts whose key/value group is the repeated
+          // field itself.
+          values = mapEntries;
+        }
+      } else if (fieldValue.constructor === Array) {
+        values = fieldValue;
       } else {
-        values.push(record[field.name]);
+        values.push(fieldValue);
       }
     }
     // check values
@@ -143,6 +167,28 @@ function shredRecordFields(
   }
 }
 
+/** Converts supported JavaScript map representations to Parquet map entries. */
+function normalizeMapEntries(value: unknown): Array<{key: unknown; value: unknown}> {
+  if (value instanceof Map) {
+    return Array.from(value, ([key, mapValue]) => ({key, value: mapValue}));
+  }
+  if (Array.isArray(value)) {
+    return value.flatMap(entry => {
+      if (Array.isArray(entry) && entry.length >= 2) {
+        return [{key: entry[0], value: entry[1]}];
+      }
+      if (entry && typeof entry === 'object') {
+        return [{key: Reflect.get(entry, 'key'), value: Reflect.get(entry, 'value')}];
+      }
+      return [];
+    });
+  }
+  if (value && typeof value === 'object') {
+    return Object.entries(value).map(([key, mapValue]) => ({key, value: mapValue}));
+  }
+  return [];
+}
+
 /**
  * 'Materialize' a list of <value, repetition_level, definition_level>
  * tuples back to nested records (objects/arrays) using the Google Dremel
@@ -173,9 +219,111 @@ export function materializeRows(schema: ParquetSchema, rowGroup: ParquetRowGroup
       materializeColumnAsRows(schema, columnData, key, rows);
     }
   }
+  decodeVariantFields(schema.fields, rows);
   return rows;
 }
 
+/** Decodes complete unshredded VARIANT groups after their binary child columns are materialized. */
+function decodeVariantFields(fields: Record<string, ParquetField>, records: unknown[]): void {
+  for (const field of Object.values(fields)) {
+    if (field.logicalType?.type === 'VARIANT') {
+      for (const record of records) {
+        if (record && typeof record === 'object') {
+          const row = record as Record<string, unknown>;
+          if (Object.prototype.hasOwnProperty.call(row, field.name)) {
+            row[field.name] = decodeVariantRecord(row[field.name], field);
+          }
+        }
+      }
+    } else if (field.fields) {
+      for (const record of records) {
+        if (record && typeof record === 'object') {
+          decodeVariantFields(field.fields, getNestedRecords(record, field.name));
+        }
+      }
+    }
+  }
+}
+
+/** Recursively decodes one materialized Variant group, preserving shredded values as records. */
+function decodeVariantRecord(value: unknown, field?: ParquetField): unknown {
+  if (Array.isArray(value)) {
+    return value.map(item => decodeVariantRecord(item, field));
+  }
+  if (!value || typeof value !== 'object') {
+    return value;
+  }
+  const record = value as Record<string, unknown>;
+  const metadata = record.metadata;
+  const variantValue = record.value;
+  if (isByteArray(metadata) && isByteArray(variantValue)) {
+    return decodeVariant(metadata, variantValue);
+  }
+  // Shredded Variant encodings may materialize the typed_value child directly while
+  // omitting the unshredded metadata/value pair. Preserve that typed representation as
+  // the logical value instead of leaking the physical wrapper into object rows.
+  if (Object.prototype.hasOwnProperty.call(record, 'typed_value')) {
+    return decodeShreddedVariantValue(record.typed_value, field?.fields?.typed_value);
+  }
+  return value;
+}
+
+/** Collapses a declared one-field typed-value union used by shredded Variant layouts. */
+function decodeShreddedVariantValue(value: unknown, field?: ParquetField): unknown {
+  if (!value || typeof value !== 'object' || Array.isArray(value)) {
+    return value;
+  }
+  const record = value as Record<string, unknown>;
+  const keys = Object.keys(record).filter(key => record[key] !== undefined);
+  if (keys.length === 1 && isVariantTypedValueField(field?.fields?.[keys[0]])) {
+    return decodeShreddedVariantValue(record[keys[0]], field?.fields?.[keys[0]]);
+  }
+  return value;
+}
+
+/** Identifies the declared members of the Parquet Variant typed-value union. */
+function isVariantTypedValueField(field: ParquetField | undefined): boolean {
+  return Boolean(field && VARIANT_TYPED_VALUE_FIELD_NAMES.has(field.name));
+}
+
+const VARIANT_TYPED_VALUE_FIELD_NAMES = new Set([
+  'null_value',
+  'boolean_value',
+  'int8_value',
+  'int16_value',
+  'int32_value',
+  'int64_value',
+  'uint8_value',
+  'uint16_value',
+  'uint32_value',
+  'uint64_value',
+  'float16_value',
+  'float32_value',
+  'float64_value',
+  'decimal4_value',
+  'decimal8_value',
+  'decimal16_value',
+  'string_value',
+  'blob_value',
+  'date_value',
+  'timestamp_millis_value',
+  'timestamp_micros_value',
+  'timestamp_nanos_value',
+  'object_value',
+  'array_value'
+]);
+
+/** Finds nested records while preserving repeated group values. */
+function getNestedRecords(record: object, fieldName: string): unknown[] {
+  const value = Reflect.get(record, fieldName);
+  return Array.isArray(value) ? value : value && typeof value === 'object' ? [value] : [];
+}
+
+/** Identifies binary values emitted by Parquet primitive materialization. */
+function isByteArray(value: unknown): value is ArrayBuffer | ArrayBufferView {
+  return value instanceof ArrayBuffer || ArrayBuffer.isView(value);
+}
+
 /** Populate record fields for one column */
 // eslint-disable-next-line max-statements, complexity
 function materializeColumnAsRows(
@@ -199,13 +347,20 @@ function materializeColumnAsRows(
   const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
   let vIndex = 0;
   for (let i = 0; i < columnData.count; i++) {
-    const dLevel = columnData.dlevels[i];
-    const rLevel = columnData.rlevels[i];
+    // Zero maximum levels are implicit and may use allocation-free empty buffers. Preserve both
+    // fallbacks: required nested scalars still need dLevel === 0 to materialize their leaf value.
+    const dLevel = columnData.dlevels[i] ?? 0;
+    const rLevel = columnData.rlevels[i] ?? 0;
     rLevels[rLevel]++;
     rLevels.fill(0, rLevel + 1);
 
     let rIndex = 0;
     let record = rows[rLevels[rIndex++] - 1];
+    if (!record) {
+      throw new Error(
+        `Parquet column ${key} referenced row ${rLevels[0] - 1} of ${rows.length} at value ${i} (rLevel ${rLevel}, dLevel ${dLevel})`
+      );
+    }
 
     // Internal nodes - Build a nested row object
     for (const step of branch) {
@@ -235,7 +390,7 @@ function materializeColumnAsRows(
 
     // Leaf node - Add the value
     if (dLevel === field.dLevelMax) {
-      const primitiveValue = columnData.values[vIndex];
+      const primitiveValue = getDecodedColumnValue(columnData, vIndex);
       const value = fromPrimitive ? fromPrimitive(primitiveValue, field) : primitiveValue;
       vIndex++;
 
@@ -273,14 +428,24 @@ function materializeFlatColumnAsRows(
 
   if (field.repetitionType === 'REQUIRED' && !fromPrimitive) {
     for (let rowIndex = 0; rowIndex < count; rowIndex++) {
-      rows[rowIndex][field.name] = columnData.values[rowIndex];
+      rows[rowIndex][field.name] = getDecodedColumnValue(columnData, rowIndex);
+    }
+    return;
+  }
+
+  if (field.repetitionType === 'REQUIRED') {
+    for (let rowIndex = 0; rowIndex < count; rowIndex++) {
+      rows[rowIndex][field.name] = fromPrimitive!(
+        getDecodedColumnValue(columnData, rowIndex),
+        field
+      );
     }
     return;
   }
 
   for (let rowIndex = 0; rowIndex < count; rowIndex++) {
     if (columnData.dlevels[rowIndex] === field.dLevelMax) {
-      const primitiveValue = columnData.values[valueIndex];
+      const primitiveValue = getDecodedColumnValue(columnData, valueIndex);
       rows[rowIndex][field.name] = fromPrimitive
         ? fromPrimitive(primitiveValue, field)
         : primitiveValue;
@@ -318,12 +483,54 @@ export function materializeColumns(
   for (const key in rowGroup.columnData) {
     const column = materializeColumn(schema, rowGroup, key);
     if (column) {
-      columns[schema.findFieldBranch(key)[0].name] = column;
+      const columnName = schema.findFieldBranch(key)[0].name;
+      columns[columnName] = columns[columnName]
+        ? mergeMaterializedColumn(columns[columnName], column)
+        : column;
     }
   }
   return columns;
 }
 
+/** Merges independently decoded nested leaf columns into one top-level column. */
+function mergeMaterializedColumn(existing: ArrayType, incoming: ArrayType): ArrayType {
+  const length = Math.max(existing.length, incoming.length);
+  const merged = new Array(length);
+  for (let index = 0; index < length; index++) {
+    merged[index] = mergeMaterializedValue(existing[index], incoming[index]);
+  }
+  return merged;
+}
+
+/** Recursively combines objects and corresponding repeated elements without losing scalar leaves. */
+function mergeMaterializedValue(existing: unknown, incoming: unknown): unknown {
+  if (existing === null || existing === undefined) {
+    return incoming;
+  }
+  if (incoming === null || incoming === undefined) {
+    return existing;
+  }
+  if (Array.isArray(existing) && Array.isArray(incoming)) {
+    const length = Math.max(existing.length, incoming.length);
+    return Array.from({length}, (_, index) =>
+      mergeMaterializedValue(existing[index], incoming[index])
+    );
+  }
+  if (
+    typeof existing === 'object' &&
+    typeof incoming === 'object' &&
+    !ArrayBuffer.isView(existing) &&
+    !ArrayBuffer.isView(incoming)
+  ) {
+    const merged: Record<string, unknown> = {...(existing as Record<string, unknown>)};
+    for (const [key, value] of Object.entries(incoming as Record<string, unknown>)) {
+      merged[key] = mergeMaterializedValue(merged[key], value);
+    }
+    return merged;
+  }
+  return incoming;
+}
+
 /** Materializes one decoded Parquet column into its top-level columnar representation. */
 export function materializeColumn(
   schema: ParquetSchema,
@@ -411,7 +618,7 @@ function materializeColumnAsColumnarArray(
       const value = Types.fromPrimitive(
         // @ts-ignore
         field.originalType || field.primitiveType,
-        columnData.values[vIndex],
+        getDecodedColumnValue(columnData, vIndex),
         field
       );
       vIndex++;
@@ -463,14 +670,34 @@ function materializeFlatColumn(
     return columnData.values;
   }
 
+  if (field.repetitionType === 'REQUIRED') {
+    const column = new Array(rowCount).fill(null);
+    for (let rowIndex = 0; rowIndex < count; rowIndex++) {
+      column[rowIndex] = fromPrimitive
+        ? fromPrimitive(getDecodedColumnValue(columnData, rowIndex), field)
+        : getDecodedColumnValue(columnData, rowIndex);
+    }
+    return column;
+  }
+
   const column = new Array(rowCount).fill(null);
   let valueIndex = 0;
   for (let rowIndex = 0; rowIndex < count; rowIndex++) {
     if (columnData.dlevels[rowIndex] === field.dLevelMax) {
-      const primitiveValue = columnData.values[valueIndex];
+      const primitiveValue = getDecodedColumnValue(columnData, valueIndex);
       column[rowIndex] = fromPrimitive ? fromPrimitive(primitiveValue, field) : primitiveValue;
       valueIndex++;
     }
   }
   return column;
 }
+
+/** Returns one compact byte value or one value from the decoder's conventional destination. */
+function getDecodedColumnValue(columnData: ParquetColumnChunk, valueIndex: number): unknown {
+  const byteArrayData = columnData.byteArrayData;
+  if (!byteArrayData) return columnData.values[valueIndex];
+  return byteArrayData.data.subarray(
+    byteArrayData.valueOffsets[valueIndex],
+    byteArrayData.valueOffsets[valueIndex + 1]
+  );
+}

package/src/parquetjs/schema/types.ts

@@ -445,13 +445,19 @@ function toPrimitive_UINT64(value: unknown): bigint {
   return BigInt.asIntN(64, primitiveValue);
 }
 
-function toPrimitive_INT96(value: any) {
-  const v = parseInt(value, 10);
-  if (Number.isNaN(v)) {
+function toPrimitive_INT96(value: unknown): number | bigint {
+  if (typeof value === 'bigint') return value;
+  if (typeof value === 'number') {
+    if (Number.isSafeInteger(value)) return value;
     throw new Error(`invalid value for INT96: ${value}`);
   }
-
-  return v;
+  if (typeof value !== 'string' || !/^[+-]?\d+$/.test(value)) {
+    throw new Error(`invalid value for INT96: ${value}`);
+  }
+  const bigintValue = BigInt(value);
+  return bigintValue >= Number.MIN_SAFE_INTEGER && bigintValue <= Number.MAX_SAFE_INTEGER
+    ? Number(bigintValue)
+    : bigintValue;
 }
 
 function toPrimitive_BYTE_ARRAY(value: any): Uint8Array {
@@ -700,24 +706,9 @@ function decimalFromPrimitive_INT(value: any, field: ParquetField) {
 }
 
 function decimalFromPrimitive_BYTE_ARRAY(value: any, field: ParquetField) {
-  let number = 0;
-  if (value.length <= 4) {
-    // Bytewise operators faster. Use them if it is possible
-    for (let i = 0; i < value.length; i++) {
-      // `value.length - i - 1` bytes have reverse order (big-endian)
-      const component = value[i] << (8 * (value.length - i - 1));
-      number += component;
-    }
-  } else {
-    for (let i = 0; i < value.length; i++) {
-      // `value.length - i - 1` bytes have reverse order (big-endian)
-      const component = value[i] * 2 ** (8 * (value.length - 1 - i));
-      number += component;
-    }
-  }
-
-  const presisionInt = Math.round(
-    ((number * 10 ** -field.presision!) % 1) * 10 ** field.presision!
-  );
-  return presisionInt * 10 ** -(field.scale || 0);
+  const bytes = toUint8Array(value);
+  let unscaledValue = 0n;
+  for (const byte of bytes) unscaledValue = (unscaledValue << 8n) | BigInt(byte);
+  if (bytes.length > 0 && (bytes[0] & 0x80) !== 0) unscaledValue -= 1n << BigInt(bytes.length * 8);
+  return Number(unscaledValue) / 10 ** (field.scale || 0);
 }