@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/src/parquet-source-types.ts

@@ -8,10 +8,22 @@ import type {
   DataSourceOptions,
   RangeRequestScheduler,
   RangeRequestSchedulerProps,
+  ScanExecutionTelemetryCallback,
   StrictLoaderOptions
 } from '@loaders.gl/loader-utils';
 import type {ArrowTableBatch, Schema} from '@loaders.gl/schema';
+import type {
+  ColumnarComparisonPredicate,
+  ColumnarInPredicate,
+  ColumnarLogicalPredicate,
+  ColumnarNotPredicate,
+  ColumnarNullPredicate,
+  ColumnarPredicateProperty,
+  ColumnarPredicateValue
+} from '@loaders.gl/loader-utils';
+import type {TableQueryExplain} from '@loaders.gl/loader-utils';
 import type {FileMetaData} from './parquetjs/parquet-thrift/index';
+import type {ParquetKeyRetriever} from './lib/parquet-encryption';
 
 /** Version validators captured from an HTTP Parquet object. */
 export type ParquetObjectVersion = {
@@ -37,6 +49,44 @@ export type ParquetColumnChunkStatistics = {
   maxIsExact?: boolean;
 };
 
+/** Size statistics reported by a Parquet writer for one column chunk. */
+export type ParquetColumnChunkSizeStatistics = {
+  /** Unencoded bytes for BYTE_ARRAY values, excluding four-byte lengths. */
+  readonly unencodedByteArrayDataBytes?: number;
+  /** Counts observed at each repetition level, when supplied. */
+  readonly repetitionLevelHistogram?: readonly number[];
+  /** Counts observed at each definition level, when supplied. */
+  readonly definitionLevelHistogram?: readonly number[];
+};
+
+/** Native Parquet geospatial bounding-box statistics for a geometry column chunk. */
+export type ParquetGeospatialBoundingBox = {
+  /** Minimum x coordinate. For GEOGRAPHY this may exceed `xmax` across the antimeridian. */
+  readonly xmin: number;
+  /** Maximum x coordinate. */
+  readonly xmax: number;
+  /** Minimum y coordinate. */
+  readonly ymin: number;
+  /** Maximum y coordinate. */
+  readonly ymax: number;
+  /** Minimum z coordinate when at least one finite z value is present. */
+  readonly zmin?: number;
+  /** Maximum z coordinate when at least one finite z value is present. */
+  readonly zmax?: number;
+  /** Minimum measure when at least one finite m value is present. */
+  readonly mmin?: number;
+  /** Maximum measure when at least one finite m value is present. */
+  readonly mmax?: number;
+};
+
+/** Native Parquet geospatial statistics for one geometry column chunk. */
+export type ParquetGeospatialStatistics = {
+  /** Coordinate bounds aggregated across non-null geometry values. */
+  readonly bbox?: ParquetGeospatialBoundingBox;
+  /** Distinct ISO WKB geometry type codes present in the column chunk. */
+  readonly geometryTypes?: readonly number[];
+};
+
 /** Normalized metadata for one Parquet column chunk. */
 export type ParquetColumnChunkMetadata = {
   /** Nested column path in the Parquet schema. */
@@ -71,8 +121,26 @@ export type ParquetColumnChunkMetadata = {
   readonly offsetIndexOffset?: number;
   /** Serialized byte length of the optional data-page location index. */
   readonly offsetIndexByteLength?: number;
+  /** Absolute file offset of the optional split-block Bloom filter. */
+  readonly bloomFilterOffset?: number;
+  /** Serialized byte length of the optional split-block Bloom filter. */
+  readonly bloomFilterByteLength?: number;
   /** Optional min/max and count statistics decoded from the footer. */
   readonly statistics?: ParquetColumnChunkStatistics;
+  /** Optional decoded size statistics for memory and nested-value planning. */
+  readonly sizeStatistics?: ParquetColumnChunkSizeStatistics;
+  /** Native geospatial statistics decoded from a Parquet 2.11+ footer. */
+  readonly geospatialStatistics?: ParquetGeospatialStatistics;
+};
+
+/** Sort-order declaration attached to one Parquet row group. */
+export type ParquetSortingColumn = {
+  /** Leaf-column index in the file schema. */
+  readonly columnIndex: number;
+  /** Whether values are sorted in descending order. */
+  readonly descending: boolean;
+  /** Whether nulls sort before non-null values. */
+  readonly nullsFirst: boolean;
 };
 
 /** Normalized metadata for one Parquet row group. */
@@ -93,6 +161,8 @@ export type ParquetRowGroupMetadata = {
   readonly compressedSize: number;
   /** Column chunks contained in the row group. */
   readonly columns: readonly ParquetColumnChunkMetadata[];
+  /** Optional sort order declared by the writer. */
+  readonly sortingColumns?: readonly ParquetSortingColumn[];
 };
 
 /** Dataset-level metadata returned by `ParquetSource.getMetadata()`. */
@@ -134,51 +204,39 @@ export type ParquetMetadataRequestOptions = {
 };
 
 /** Scalar values supported by exact Parquet source predicates. */
-export type ParquetPredicateValue = boolean | number | bigint | string | Date | Uint8Array;
+export type ParquetPredicateValue = ColumnarPredicateValue;
 
-/** Reference to one top-level Parquet column in a predicate expression. */
-export type ParquetPredicateProperty = {
-  /** Top-level column name. */
-  property: string;
-};
+/** Four-, six-, or eight-dimensional extent used for conservative Parquet spatial pruning. */
+export type ParquetBoundingBox =
+  | readonly [number, number, number, number]
+  | readonly [number, number, number, number, number, number]
+  | readonly [number, number, number, number, number, number, number, number];
+
+/** Reference to one Parquet column in a predicate expression. */
+export type ParquetPredicateProperty = ColumnarPredicateProperty;
 
 /** Comparison predicate applied to one top-level Parquet column. */
-export type ParquetComparisonPredicate = {
-  /** CQL2-shaped exact comparison operator. */
-  op: '=' | '<>' | '<' | '<=' | '>' | '>=';
-  /** Column reference followed by the scalar value to compare. */
+export type ParquetComparisonPredicate = Omit<ColumnarComparisonPredicate, 'args'> & {
   args: readonly [ParquetPredicateProperty, ParquetPredicateValue];
 };
 
 /** Membership predicate applied to one top-level Parquet column. */
-export type ParquetInPredicate = {
-  /** CQL2-shaped membership operator. */
-  op: 'in';
-  /** Column reference followed by the candidate scalar values. */
+export type ParquetInPredicate = Omit<ColumnarInPredicate, 'args'> & {
   args: readonly [ParquetPredicateProperty, readonly ParquetPredicateValue[]];
 };
 
 /** Null predicate applied to one top-level Parquet column. */
-export type ParquetNullPredicate = {
-  /** CQL2-shaped null test operator. */
-  op: 'isNull';
-  /** Column reference to test. */
+export type ParquetNullPredicate = Omit<ColumnarNullPredicate, 'args'> & {
   args: readonly [ParquetPredicateProperty];
 };
 
 /** Logical composition of serializable Parquet predicates. */
-export type ParquetLogicalPredicate = {
-  /** CQL2-shaped logical operator applied to the child predicates. */
-  op: 'and' | 'or';
-  /** At least two child predicates. */
+export type ParquetLogicalPredicate = Omit<ColumnarLogicalPredicate, 'args'> & {
   args: readonly ParquetPredicate[];
 };
 
 /** Negation of one serializable Parquet predicate. */
-export type ParquetNotPredicate = {
-  /** CQL2-shaped logical negation operator. */
-  op: 'not';
-  /** Single child predicate. */
+export type ParquetNotPredicate = Omit<ColumnarNotPredicate, 'args'> & {
   args: readonly [ParquetPredicate];
 };
 
@@ -201,6 +259,8 @@ export type ParquetSourceReadOptions = {
   rowGroups?: readonly number[];
   /** Top-level columns to fetch and decode. Defaults to all columns. */
   columns?: readonly string[];
+  /** Maximum number of rows retained after filtering across all emitted batches. */
+  limit?: number;
   /** Maximum number of rows in each emitted Arrow batch. Defaults to one row group per batch. */
   batchSize?: number;
   /** Maximum number of row groups decoded concurrently. */
@@ -209,10 +269,73 @@ export type ParquetSourceReadOptions = {
   rowGroupFilter?: (rowGroup: ParquetRowGroupMetadata) => boolean;
   /** Serializable exact row predicate, conservatively pushed into row-group statistics. */
   predicate?: ParquetPredicate;
+  /** Spatial extent pruned with native statistics or a GeoParquet 1.1 bbox covering when present. */
+  bbox?: ParquetBoundingBox;
+  /** Geometry column whose GeoParquet covering should serve `bbox`; defaults to `primary_column`. */
+  geometryColumn?: string;
   /** Abort this read and all of its outstanding range requests. */
   signal?: AbortSignal;
+  /** Receives one portable terminal execution snapshot for this read. */
+  onTelemetry?: ScanExecutionTelemetryCallback;
 };
 
+/** Explain result for a Parquet table query, including footer-level row-group pruning. */
+export type ParquetSourceExplain = TableQueryExplain<ParquetPredicate> &
+  Readonly<{
+    /** Physical source kind. */
+    source: 'parquet';
+    /** Row groups selected by the request and conservative footer statistics. */
+    rowGroups: Readonly<{
+      /** Zero-based row-group indexes retained by every physical pruning stage. */
+      indices: readonly number[];
+      requested: number;
+      selected: number;
+      /** Row groups rejected by the caller-supplied metadata callback. */
+      prunedByCallback: number;
+      /** Row groups rejected by file-native geospatial statistics. */
+      prunedBySpatial: number;
+      prunedByStatistics: number;
+      /** Row groups proven absent by split-block Bloom filters. */
+      prunedByBloomFilter: number;
+    }>;
+    /** Bloom-filter ranges inspected while producing the physical plan. */
+    bloomFilters: Readonly<{
+      read: number;
+      bytesRead: number;
+    }>;
+    /** Selective page-index plans available for retained row groups. */
+    pages: Readonly<{
+      rowGroupsPlanned: number;
+      indexesRead: number;
+      total: number;
+      selected: number;
+      rowsPruned: number;
+      plans: readonly ParquetPageScanPlan[];
+    }>;
+  }>;
+
+/** Explainable page-index and data-range plan for one Parquet row group. */
+export type ParquetPageScanPlan = Readonly<{
+  /** Zero-based row-group index. */
+  rowGroupIndex: number;
+  /** Physical phase represented by this plan. */
+  phase: 'combined' | 'predicate' | 'projection';
+  /** Nested column paths fetched during this physical phase. */
+  columns: readonly (readonly string[])[];
+  /** Candidate half-open row ranges retained by page statistics. */
+  rowRanges: readonly Readonly<{start: number; end: number}>[];
+  /** Column-index and offset-index payloads decoded. */
+  indexesRead: number;
+  /** Data pages represented by the selected columns. */
+  totalPages: number;
+  /** Data pages intersecting retained row ranges. */
+  selectedPages: number;
+  /** Logical rows excluded before data-page decoding. */
+  rowsPruned: number;
+  /** Absolute file ranges required for the retained data pages. */
+  ranges: readonly Readonly<{offset: number; length: number}>[];
+}>;
+
 /** Compatibility alias for selective source read options. */
 export type ParquetReadOptions = ParquetSourceReadOptions;
 
@@ -244,6 +367,12 @@ export type ParquetTelemetry = {
   rowGroupsPruned: number;
   /** Candidate row groups proven impossible using footer statistics. */
   rowGroupsPrunedByStatistics: number;
+  /** Candidate row groups proven impossible using split-block Bloom filters. */
+  rowGroupsPrunedByBloomFilter: number;
+  /** Bloom-filter payloads fetched for selective reads. */
+  bloomFiltersRead: number;
+  /** Bytes fetched for Bloom-filter payloads. */
+  bloomFilterBytesRead: number;
   /** Row groups proven impossible using page-level column indexes. */
   rowGroupsPrunedByPageIndex: number;
   /** Column-index and offset-index blobs decoded for selective reads. */
@@ -277,6 +406,7 @@ export type ParquetTelemetryEvent = {
     | 'range-request'
     | 'cache-hit'
     | 'row-group-prune'
+    | 'bloom-filter'
     | 'page-index-prune'
     | 'predicate-filter'
     | 'decode'
@@ -346,6 +476,16 @@ export type ParquetSourceLoaderOptions = DataSourceOptions & {
     headers?: HeadersInit;
     /** Preserve binary values when the TypeScript decoder is used for later reads. */
     preserveBinary?: boolean;
+    /** Decode legacy INT96 values as epoch-nanosecond timestamps. */
+    int96AsTimestamp?: boolean;
+    /** Verify page-header CRC values in TypeScript reads. */
+    verifyPageChecksums?: boolean;
+    /** Resolves modular-encryption keys from file and column key metadata. */
+    keyRetriever?: ParquetKeyRetriever;
+    /** AAD prefix for encrypted files that intentionally omit it from metadata. */
+    aadPrefix?: Uint8Array;
+    /** Verify plaintext-footer signatures when present. Enabled by default. */
+    verifyFooterSignature?: boolean;
     /** Receives cumulative transport, pruning, decode, and batch telemetry events. */
     onTelemetry?: (event: ParquetTelemetryEvent) => void;
     /** Overrides the package-local worker used for selective source decoding. */
@@ -357,10 +497,11 @@ export type ParquetSourceLoaderOptions = DataSourceOptions & {
   rangeRequests?: ParquetRangeRequestOptions;
 };
 
-/** Four- or six-dimensional extent used for conservative Parquet dataset file pruning. */
+/** Four-, six-, or eight-dimensional extent used for conservative Parquet dataset file pruning. */
 export type ParquetDatasetBoundingBox =
   | readonly [number, number, number, number]
-  | readonly [number, number, number, number, number, number];
+  | readonly [number, number, number, number, number, number]
+  | readonly [number, number, number, number, number, number, number, number];
 
 /** Scalar value carried by a partitioned Parquet dataset file descriptor. */
 export type ParquetDatasetPartitionValue = string | number | boolean | null;
@@ -421,6 +562,34 @@ export type ParquetDatasetReadOptions = Omit<ParquetSourceReadOptions, 'rowGroup
     fileConcurrency?: number;
   };
 
+/** One retained file and its child Parquet physical scan plan. */
+export type ParquetDatasetFileScanPlan = Readonly<{
+  /** Position in provider output before descriptor pruning. */
+  fileIndex: number;
+  /** Stable descriptor or source identifier. */
+  fileId: string;
+  /** Descriptor partition values used during file pruning. */
+  partitions?: Readonly<Record<string, ParquetDatasetPartitionValue>>;
+  /** Physical row-group and Bloom-filter plan for this file. */
+  parquet: ParquetSourceExplain;
+}>;
+
+/** Common logical query explanation plus physical multi-file Parquet planning. */
+export type ParquetDatasetExplain = TableQueryExplain<ParquetPredicate> &
+  Readonly<{
+    /** Physical source kind. */
+    source: 'parquet-dataset';
+    /** Descriptor discovery and file-pruning summary. */
+    files: Readonly<{
+      discovered: number;
+      selected: number;
+      prunedByBoundingBox: number;
+      prunedByPartitions: number;
+    }>;
+    /** Ordered physical plans for retained files. */
+    filePlans: readonly ParquetDatasetFileScanPlan[];
+  }>;
+
 /** Dataset and file provenance attached to an emitted Arrow batch. */
 export type ParquetDatasetBatchProvenance = ParquetBatchProvenance & {
   /** Zero-based descriptor position in provider output, before local pruning. */

package/src/parquet-worker-url.ts

@@ -4,11 +4,16 @@
 
 /** Module-relative worker asset for browser ESM and Node.js CommonJS distributions. */
 export const PARQUET_WORKER_URL = import.meta.url
-  ? new URL(getParquetWorkerFile(), import.meta.url).toString()
+  ? getParquetWorkerUrl(import.meta.url)
   : typeof __dirname === 'string'
     ? `${__dirname}/parquet-worker-node.cjs`
     : true;
 
+/** Builds a package-relative worker URL without making the development bundler resolve the asset. */
+function getParquetWorkerUrl(moduleUrl: string): string {
+  return `${moduleUrl.slice(0, moduleUrl.lastIndexOf('/') + 1)}${getParquetWorkerFile()}`;
+}
+
 /** Selects the worker bundle for the current JavaScript runtime. */
 function getParquetWorkerFile(): string {
   const isNode = Boolean(

package/src/parquetjs/codecs/declare.ts

@@ -17,6 +17,29 @@ export type ParquetValueBuffer =
   | Float32Array
   | Float64Array;
 
+/** Mutable Arrow-compatible destination for compact physical BYTE_ARRAY values. */
+export type ParquetByteArrayOutput = {
+  /** Contiguous bytes copied from decoded physical values. */
+  data: Uint8Array;
+  /** Arrow-compatible offsets for compact non-null physical values. */
+  valueOffsets: Int32Array;
+  /** Number of bytes written to `data`. */
+  byteLength: number;
+};
+
+/** Ensures a compact byte destination can accept an additional physical value. */
+export function reserveParquetByteArrayOutput(
+  output: ParquetByteArrayOutput,
+  additionalByteLength: number
+): void {
+  const requiredByteLength = output.byteLength + additionalByteLength;
+  if (requiredByteLength <= output.data.byteLength) return;
+  const nextByteLength = Math.max(requiredByteLength, output.data.byteLength * 2, 1024);
+  const data = new Uint8Array(nextByteLength);
+  data.set(output.data.subarray(0, output.byteLength));
+  output.data = data;
+}
+
 export interface CursorBuffer {
   buffer: Uint8Array;
   offset: number;
@@ -35,8 +58,12 @@ export interface ParquetCodecOptions {
   outputOffset?: number;
   /** Optional dictionary resolved while decoding dictionary indices. */
   dictionary?: readonly unknown[];
+  /** Optional compact destination used by PLAIN BYTE_ARRAY decoding. */
+  byteArrayOutput?: ParquetByteArrayOutput;
   /** Preserve decoded INT64 values as bigint instead of converting them to number. */
   int64AsBigInt?: boolean;
+  /** Decode legacy INT96 physical values as epoch nanoseconds instead of raw numbers. */
+  int96AsTimestamp?: boolean;
 }
 
 export interface ParquetCodecKit {