@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/src/iceberg-types.ts

@@ -0,0 +1,165 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Minimal Iceberg table metadata shape needed for read-only table discovery. */
+export type IcebergTableMetadata = {
+  /** Iceberg metadata format version. */
+  readonly 'format-version': 1 | 2 | 3;
+  /** Table UUID when supplied by the writer. */
+  readonly 'table-uuid'?: string;
+  /** Root location used to resolve manifest and data-file paths. */
+  readonly location: string;
+  /** Current snapshot identifier, or `-1` when the table has no snapshot. */
+  readonly 'current-snapshot-id'?: number;
+  /** Snapshot sequence number. */
+  readonly 'last-sequence-number'?: number;
+  /** Metadata update time in milliseconds since the Unix epoch. */
+  readonly 'last-updated-ms'?: number;
+  /** Current schema identifier. */
+  readonly 'current-schema-id'?: number;
+  /** Schemas declared by the table metadata. */
+  readonly schemas?: readonly IcebergSchema[];
+  /** Partition specifications declared by the table metadata. */
+  readonly 'partition-specs'?: readonly IcebergPartitionSpec[];
+  /** Default partition specification identifier. */
+  readonly 'default-spec-id'?: number;
+  /** Table snapshots. */
+  readonly snapshots?: readonly IcebergSnapshot[];
+  /** Named snapshot references such as `main`, branches, and tags. */
+  readonly refs?: Readonly<Record<string, IcebergSnapshotReference>>;
+  /** Table properties. */
+  readonly properties?: Readonly<Record<string, string>>;
+  /** Additional Iceberg metadata fields retained for forward compatibility. */
+  readonly [key: string]: unknown;
+};
+
+/** Named Iceberg branch or tag reference. */
+export type IcebergSnapshotReference = Readonly<Record<string, unknown>> & {
+  readonly 'snapshot-id': number;
+  readonly type?: 'branch' | 'tag' | string;
+};
+
+/** Iceberg schema metadata retained without imposing a schema-model dependency. */
+export type IcebergSchema = Readonly<Record<string, unknown>> & {
+  readonly 'schema-id'?: number;
+  readonly fields?: readonly Readonly<Record<string, unknown>>[];
+};
+
+/** Iceberg partition specification metadata. */
+export type IcebergPartitionSpec = Readonly<Record<string, unknown>> & {
+  readonly 'spec-id'?: number;
+  readonly fields?: readonly Readonly<Record<string, unknown>>[];
+};
+
+/** Iceberg snapshot metadata used to select a manifest list. */
+export type IcebergSnapshot = Readonly<Record<string, unknown>> & {
+  readonly 'snapshot-id': number;
+  readonly 'sequence-number'?: number;
+  readonly 'manifest-list'?: string;
+  readonly 'schema-id'?: number;
+  readonly summary?: Readonly<Record<string, string>>;
+};
+
+/** One data or delete manifest referenced by an Iceberg manifest list. */
+export type IcebergManifestFile = Readonly<Record<string, unknown>> & {
+  readonly manifest_path: string;
+  readonly manifest_length?: number;
+  readonly partition_spec_id?: number;
+  readonly content?: number | string;
+};
+
+/** One data or delete file entry decoded from an Iceberg manifest. */
+export type IcebergDataFile = Readonly<Record<string, unknown>> & {
+  readonly file_path: string;
+  readonly file_format: string;
+  readonly file_size_in_bytes?: number;
+  readonly record_count?: number;
+  readonly partition?: Readonly<Record<string, unknown>>;
+  readonly lower_bounds?: Readonly<Record<string, unknown>>;
+  readonly upper_bounds?: Readonly<Record<string, unknown>>;
+  readonly content?: number | string;
+  readonly referenced_data_file?: string;
+  readonly equality_ids?: readonly number[];
+  readonly sort_order_id?: number;
+  readonly data_sequence_number?: number;
+};
+
+/** Parquet data-file plan produced by the Iceberg manifest reader. */
+export type IcebergParquetFile = {
+  readonly data: string;
+  readonly fileSize?: number;
+  readonly recordCount?: number;
+  readonly partition?: Readonly<Record<string, unknown>>;
+  readonly lowerBounds?: Readonly<Record<string, unknown>>;
+  readonly upperBounds?: Readonly<Record<string, unknown>>;
+  /** Manifest path that contributed this file. */
+  readonly manifestPath?: string;
+  /** Partition specification identifier for this manifest. */
+  readonly partitionSpecId?: number;
+  /** Snapshot used to produce this plan. */
+  readonly snapshotId?: number;
+  /** Iceberg data sequence number when supplied by the manifest entry. */
+  readonly dataSequenceNumber?: number;
+  /** Schema identifier used by the snapshot that contributed this file. */
+  readonly schemaId?: number;
+};
+
+/** Planned Iceberg delete file retained for optional delete application during a scan. */
+export type IcebergDeleteFile = {
+  /** URL or path of the delete file. */
+  readonly data: string;
+  /** Iceberg delete-file format, commonly `avro` or `puffin`. */
+  readonly format: string;
+  /** Delete content kind, such as position or equality deletes. */
+  readonly content?: number | string;
+  /** Referenced data file for a position delete file. */
+  readonly referencedDataFile?: string;
+  /** Field IDs used by an equality delete file. */
+  readonly equalityFieldIds?: readonly number[];
+  /** File size from the manifest. */
+  readonly fileSize?: number;
+  /** Record count from the manifest. */
+  readonly recordCount?: number;
+  /** Partition values from the manifest. */
+  readonly partition?: Readonly<Record<string, unknown>>;
+  /** Manifest path that contributed this delete file. */
+  readonly manifestPath?: string;
+  /** Partition specification identifier for this manifest. */
+  readonly partitionSpecId?: number;
+  /** Snapshot used to produce this plan. */
+  readonly snapshotId?: number;
+  /** Schema identifier used by the snapshot that contributed this delete file. */
+  readonly schemaId?: number;
+  /** Iceberg data sequence number associated with this delete file. */
+  readonly dataSequenceNumber?: number;
+};
+
+/** Files selected from one Iceberg snapshot before delete application. */
+export type IcebergScanPlan = {
+  /** Active Parquet data files selected from data manifests. */
+  readonly dataFiles: readonly IcebergParquetFile[];
+  /** Delete files discovered from delete manifests. */
+  readonly deleteFiles: readonly IcebergDeleteFile[];
+  /** Snapshot identifier used to build the plan. */
+  readonly snapshotId?: number;
+  /** Named snapshot reference used to build this plan, when supplied. */
+  readonly snapshotRef?: string;
+};
+
+/** Axis-aligned geographic bounds in `[minX, minY, maxX, maxY]` order. */
+export type IcebergBoundingBox = readonly [number, number, number, number];
+
+/** Conservative spatial filter used for Iceberg file-level pruning. */
+export type IcebergSpatialFilter = {
+  /** Top-level geometry column or Iceberg field name whose bounds are being queried. */
+  readonly column: string;
+  /** Query envelope in `[minX, minY, maxX, maxY]` order. */
+  readonly bbox: IcebergBoundingBox;
+};
+
+/** Options for constructing a read-only Iceberg table source. */
+export type IcebergTableSourceOptions = {
+  /** HTTP headers forwarded to metadata requests. */
+  readonly headers?: HeadersInit;
+};

package/src/index.ts

@@ -16,13 +16,19 @@ export type {
   ParquetBatch,
   ParquetBatchMetadata,
   ParquetBatchProvenance,
+  ParquetBoundingBox,
   ParquetColumnChunkMetadata,
   ParquetColumnChunkStatistics,
+  ParquetColumnChunkSizeStatistics,
+  ParquetGeospatialBoundingBox,
+  ParquetGeospatialStatistics,
   ParquetDatasetBatch,
   ParquetDatasetBatchProvenance,
   ParquetDatasetBoundingBox,
+  ParquetDatasetExplain,
   ParquetDatasetFile,
   ParquetDatasetFileCollection,
+  ParquetDatasetFileScanPlan,
   ParquetDatasetFileProvider,
   ParquetDatasetFileQuery,
   ParquetDatasetFiles,
@@ -32,6 +38,7 @@ export type {
   ParquetDatasetTelemetry,
   ParquetMetadataRequestOptions,
   ParquetObjectVersion,
+  ParquetPageScanPlan,
   ParquetComparisonPredicate,
   ParquetInPredicate,
   ParquetLogicalPredicate,
@@ -43,22 +50,49 @@ export type {
   ParquetRangeRequestOptions,
   ParquetReadOptions,
   ParquetRowGroupMetadata,
+  ParquetSortingColumn,
   ParquetSourceBatch,
   ParquetSourceLoaderOptions,
   ParquetSourceMetadata,
   ParquetSourceReadOptions,
+  ParquetSourceExplain,
   ParquetTelemetry,
   ParquetTelemetryEvent
 } from './parquet-source-types';
 export {
+  PARQUET_TABLE_QUERY_CAPABILITIES,
   PARQUET_SOURCE_CAPABILITIES,
   type ParquetSourceCapabilities
 } from './parquet-source-capabilities';
 
 export {ParquetWriter} from './parquet-writer';
-export type {ParquetJSWriterEncoding, ParquetJSWriterOptions} from './parquet-js-writer';
+export type {
+  ParquetJSWriterEncoding,
+  ParquetJSWriterOptions,
+  ParquetSortingColumnOption,
+  ParquetWriterEncryptionOptions,
+  ParquetWriterFooterSignatureOptions
+} from './parquet-js-writer';
 export {ParquetJSWriter} from './parquet-js-writer';
 
+export {
+  decodeParquetColumnIndex,
+  decodeParquetPageStatisticsValue,
+  canUseParquetPageIndexForColumn,
+  type ParquetPageStatistics
+} from './lib/parquet-page-index';
+export {
+  createParquetModuleAad,
+  decryptParquetModule,
+  encryptParquetModule,
+  createParquetFooterSignature,
+  readParquetEncryptedModule,
+  verifyParquetFooterSignature,
+  type ParquetDecryptModuleOptions,
+  type ParquetEncryptionAlgorithm,
+  type ParquetKeyRetriever
+} from './lib/parquet-encryption';
+
 // EXPERIMENTAL - expose the internal parquetjs API
 
 export {preloadCompressions} from './parquetjs/compression';
@@ -71,3 +105,18 @@ export {
   convertParquetSchema,
   convertParquetSchema as convertParquetToArrowSchema
 } from './lib/arrow/convert-schema-from-parquet';
+
+export {
+  checkParquetSplitBlockBloomFilter,
+  decodeParquetSplitBlockBloomFilter,
+  encodeParquetBloomFilterValue,
+  encodeParquetSplitBlockBloomFilter,
+  hashParquetBloomFilterValue,
+  insertParquetSplitBlockBloomFilter,
+  type ParquetBloomFilterPhysicalType,
+  type ParquetSplitBlockBloomFilter
+} from './lib/parquet-bloom-filter';
+export {
+  getParquetBloomFilterProbes,
+  type ParquetBloomFilterProbe
+} from './lib/parquet-bloom-filter-planner';

package/src/lib/arrow/convert-schema-from-parquet.ts

@@ -52,9 +52,10 @@ export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
 
 export function convertParquetSchema(
   parquetSchema: ParquetSchema,
-  parquetMetadata: FileMetaData | null
+  parquetMetadata: FileMetaData | null,
+  options?: {int96AsTimestamp?: boolean}
 ): Schema {
-  const fields = getFields(parquetSchema.schema);
+  const fields = getFields(parquetSchema.schema, options);
   const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
 
   const schema: Schema = {
@@ -65,31 +66,28 @@ export function convertParquetSchema(
   return schema;
 }
 
-function getFields(schema: SchemaDefinition): Field[] {
+function getFields(schema: SchemaDefinition, options?: {int96AsTimestamp?: boolean}): Field[] {
   const fields: Field[] = [];
 
   for (const name in schema) {
-    fields.push(getField(name, schema[name]));
+    fields.push(getField(name, schema[name], options));
   }
 
   return fields;
 }
 
 /** Converts one Parquet field, preserving repeated values as Arrow lists. */
-function getField(name: string, field: FieldDefinition): Field {
-  const elementField: Field = field.fields
-    ? {
-        name,
-        type: {type: 'struct', children: getFields(field.fields)},
-        nullable: Boolean(field.optional),
-        metadata: getFieldMetadata(field)
-      }
-    : {
-        name,
-        type: getFieldType(field),
-        nullable: Boolean(field.optional),
-        metadata: getFieldMetadata(field)
-      };
+function getField(
+  name: string,
+  field: FieldDefinition,
+  options?: {int96AsTimestamp?: boolean}
+): Field {
+  const elementField: Field = {
+    name,
+    type: getFieldValueType(field, options),
+    nullable: Boolean(field.optional),
+    metadata: getFieldMetadata(field)
+  };
 
   if (!field.repeated) {
     return elementField;
@@ -106,8 +104,95 @@ function getField(name: string, field: FieldDefinition): Field {
   };
 }
 
+/** Converts a nested Parquet definition into its corresponding Arrow data type. */
+function getFieldValueType(
+  field: FieldDefinition,
+  options?: {int96AsTimestamp?: boolean}
+): DataType {
+  if (!field.fields) {
+    return getFieldType(field, options);
+  }
+
+  if (field.logicalType?.type === 'LIST') {
+    const element = field.fields.list?.fields?.element || field.fields.element;
+    if (!element) {
+      // Preserve unusual legacy LIST layouts as structs. Their group names and
+      // repetition levels are still available to the row-materialization path.
+      return {type: 'struct', children: getFields(field.fields, options)};
+    }
+    if (
+      (element.logicalType?.type === 'LIST' &&
+        (!isStandardListDefinition(element) || element.optional !== false)) ||
+      (element.logicalType?.type === 'MAP' &&
+        (!isStandardMapDefinition(element) || element.optional !== false))
+    ) {
+      // Preserve nested legacy wrappers. Arrow's high-level List/Map type
+      // cannot describe the historical wrapper shape without changing the
+      // object-row contract used by existing callers.
+      return {type: 'struct', children: getFields(field.fields, options)};
+    }
+    return {
+      type: 'list',
+      children: [
+        {
+          name: 'element',
+          type: getFieldValueType(element, options),
+          nullable: Boolean(element.optional)
+        }
+      ]
+    };
+  }
+
+  if (field.logicalType?.type === 'MAP') {
+    const entry = field.fields.key_value?.fields || field.fields;
+    const key = entry?.key;
+    const value = entry?.value;
+    if (!key || !value) {
+      // Preserve unusual legacy MAP layouts as structs rather than rejecting a
+      // file whose converted annotation does not follow the standard wrapper.
+      return {type: 'struct', children: getFields(field.fields, options)};
+    }
+    return {
+      type: 'map',
+      keysSorted: false,
+      children: [
+        {
+          name: 'entries',
+          type: {
+            type: 'struct',
+            children: [
+              {name: 'key', type: getFieldValueType(key, options), nullable: false},
+              {
+                name: 'value',
+                type: getFieldValueType(value, options),
+                nullable: Boolean(value.optional)
+              }
+            ]
+          },
+          nullable: false
+        }
+      ]
+    };
+  }
+
+  return {type: 'struct', children: getFields(field.fields, options)};
+}
+
+/** Returns whether a nested LIST follows the standard list/list/element wrapper. */
+function isStandardListDefinition(field: FieldDefinition): boolean {
+  return Boolean(field.fields?.list?.fields?.element);
+}
+
+/** Returns whether a nested MAP follows the standard map/key_value/key/value wrapper. */
+function isStandardMapDefinition(field: FieldDefinition): boolean {
+  return Boolean(field.fields?.key_value?.fields?.key && field.fields.key_value.fields.value);
+}
+
 /** Returns the exact serialized Arrow type for one decoded Parquet field. */
-function getFieldType(field: FieldDefinition): DataType {
+function getFieldType(field: FieldDefinition, options?: {int96AsTimestamp?: boolean}): DataType {
+  if (field.type === 'INT96' && options?.int96AsTimestamp) {
+    return 'timestamp-nanosecond';
+  }
   if (field.logicalType?.type === 'DECIMAL') {
     const precision = field.precision ?? field.presision;
     const scale = field.scale;
@@ -129,6 +214,15 @@ function getFieldType(field: FieldDefinition): DataType {
 function getFieldMetadata(field: FieldDefinition): Record<string, string> | undefined {
   let metadata: Record<string, string> | undefined;
 
+  if (
+    field.logicalType?.type === 'VARIANT' &&
+    field.logicalType.specificationVersion !== undefined
+  ) {
+    metadata = {
+      'parquet.variant.specification_version': String(field.logicalType.specificationVersion)
+    };
+  }
+
   for (const key in field) {
     const fieldValue = field[key];
     if (

package/src/lib/arrow/create-nested-arrow-vector.ts

@@ -416,6 +416,16 @@ function createSelectedPrimitiveData(
   columnData: ParquetColumnChunk,
   selected: SelectedLeafValues
 ): arrow.Data | undefined {
+  const directData = createSelectedPrimitiveArrayView(
+    arrowType,
+    parquetField,
+    columnData,
+    selected
+  );
+  if (directData) {
+    return arrow.makeData({type: arrowType, data: directData, nullCount: 0} as any);
+  }
+
   const data = createNestedPrimitiveArray(arrowType, parquetField, selected.length);
   if (!data) {
     return undefined;
@@ -437,6 +447,75 @@ function createSelectedPrimitiveData(
   } as any);
 }
 
+/** Reuses a contiguous decoded primitive range when nested selection needs no null expansion. */
+function createSelectedPrimitiveArrayView(
+  arrowType: arrow.DataType,
+  parquetField: ParquetField,
+  columnData: ParquetColumnChunk,
+  selected: SelectedLeafValues
+): NestedPrimitiveArrowArray | undefined {
+  if (selected.nullCount || selected.length !== selected.valueCount) {
+    return undefined;
+  }
+  const values = getMatchingNestedPrimitiveArray(arrowType, parquetField, columnData.values);
+  return values?.subarray(
+    selected.firstValueIndex,
+    selected.firstValueIndex + selected.valueCount
+  ) as NestedPrimitiveArrowArray | undefined;
+}
+
+/** Returns decoded nested values when their physical array exactly matches the Arrow type. */
+function getMatchingNestedPrimitiveArray(
+  arrowType: arrow.DataType,
+  parquetField: ParquetField,
+  values: ParquetColumnChunk['values']
+): NestedPrimitiveArrowArray | undefined {
+  if (
+    parquetField.primitiveType === 'FLOAT' &&
+    arrowType instanceof arrow.Float32 &&
+    values instanceof Float32Array
+  ) {
+    return values;
+  }
+  if (
+    parquetField.primitiveType === 'DOUBLE' &&
+    arrowType instanceof arrow.Float64 &&
+    values instanceof Float64Array
+  ) {
+    return values;
+  }
+  if (parquetField.primitiveType === 'INT32') {
+    if (arrowType instanceof arrow.Int8 && values instanceof Int8Array) return values;
+    if (arrowType instanceof arrow.Int16 && values instanceof Int16Array) return values;
+    if (
+      (arrowType instanceof arrow.Int32 ||
+        arrowType instanceof arrow.DateDay ||
+        arrowType instanceof arrow.TimeMillisecond) &&
+      values instanceof Int32Array
+    ) {
+      return values;
+    }
+    if (arrowType instanceof arrow.Uint8 && values instanceof Uint8Array) return values;
+    if (arrowType instanceof arrow.Uint16 && values instanceof Uint16Array) return values;
+    if (arrowType instanceof arrow.Uint32 && values instanceof Uint32Array) return values;
+  }
+  if (parquetField.primitiveType === 'INT64') {
+    if (
+      (arrowType instanceof arrow.Int64 ||
+        arrowType instanceof arrow.TimeMicrosecond ||
+        arrowType instanceof arrow.TimeNanosecond ||
+        arrowType instanceof arrow.TimestampMillisecond ||
+        arrowType instanceof arrow.TimestampMicrosecond ||
+        arrowType instanceof arrow.TimestampNanosecond) &&
+      values instanceof BigInt64Array
+    ) {
+      return values;
+    }
+    if (arrowType instanceof arrow.Uint64 && values instanceof BigUint64Array) return values;
+  }
+  return undefined;
+}
+
 /** Allocates the Arrow array matching an unconverted physical Parquet primitive. */
 function createNestedPrimitiveArray(
   arrowType: arrow.DataType,
@@ -517,6 +596,9 @@ function createSelectedByteData(
   if (!supportsNestedByteData(arrowType, parquetField)) {
     return undefined;
   }
+  if (columnData.byteArrayData) {
+    return createSelectedBufferedByteData(arrowType, columnData, selected);
+  }
 
   const byteValues = columnData.values as Uint8Array[];
   const valueOffsets = new Int32Array(selected.length + 1);
@@ -556,6 +638,36 @@ function createSelectedByteData(
   } as any);
 }
 
+/** Creates selected nested Arrow bytes from the decoder's compact physical value buffer. */
+function createSelectedBufferedByteData(
+  arrowType: arrow.DataType,
+  columnData: ParquetColumnChunk,
+  selected: SelectedLeafValues
+): arrow.Data {
+  const byteArrayData = columnData.byteArrayData!;
+  const valueOffsets = new Int32Array(selected.length + 1);
+  let sourceValueIndex = selected.firstValueIndex;
+  let dataByteLength = 0;
+  for (let outputIndex = 0; outputIndex < selected.length; outputIndex++) {
+    if (!selected.nullBitmap || selected.nullBitmap[outputIndex >> 3] & (1 << (outputIndex & 7))) {
+      dataByteLength +=
+        byteArrayData.valueOffsets[sourceValueIndex + 1] -
+        byteArrayData.valueOffsets[sourceValueIndex];
+      sourceValueIndex++;
+    }
+    valueOffsets[outputIndex + 1] = dataByteLength;
+  }
+  const firstByteOffset = byteArrayData.valueOffsets[selected.firstValueIndex];
+  const lastByteOffset = byteArrayData.valueOffsets[selected.firstValueIndex + selected.valueCount];
+  return arrow.makeData({
+    type: arrowType,
+    valueOffsets,
+    data: byteArrayData.data.subarray(firstByteOffset, lastByteOffset),
+    nullBitmap: selected.nullBitmap,
+    nullCount: selected.nullCount
+  } as any);
+}
+
 /** Returns whether one Parquet byte leaf maps directly to Arrow Utf8 or Binary. */
 function supportsNestedByteData(arrowType: arrow.DataType, parquetField: ParquetField): boolean {
   if (arrowType instanceof arrow.Utf8) {
@@ -564,6 +676,10 @@ function supportsNestedByteData(arrowType: arrow.DataType, parquetField: Parquet
   return (
     arrowType instanceof arrow.Binary &&
     (!parquetField.originalType ||
+      parquetField.originalType === 'JSON' ||
+      parquetField.originalType === 'BSON' ||
+      parquetField.originalType === 'INTERVAL' ||
+      parquetField.originalType === 'VARIANT' ||
       parquetField.originalType === 'GEOMETRY' ||
       parquetField.originalType === 'GEOGRAPHY') &&
     (parquetField.primitiveType === 'BYTE_ARRAY' ||