@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

package/src/parquet-encoded-page-types.ts

@@ -0,0 +1,122 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {
+  ParquetBoundingBox,
+  ParquetPredicate,
+  ParquetRowGroupMetadata
+} from './parquet-source-types';
+
+/** Byte range within an encoded Parquet page body. */
+export type ParquetEncodedPageSection = Readonly<{
+  /** Byte offset relative to {@link ParquetEncodedPage.data}. */
+  byteOffset: number;
+  /** Number of bytes in this section. */
+  byteLength: number;
+}>;
+
+/** Whether the page value payload still uses the column chunk's compression codec. */
+export type ParquetPageCompressionState = 'compressed' | 'decompressed';
+
+/** Encoded Parquet page body suitable for deferred CPU or GPU decoding. */
+export type ParquetEncodedPage = Readonly<{
+  /** Page role and data-page version. */
+  type: 'dictionary' | 'data-v1' | 'data-v2';
+  /** Zero-based data-page ordinal within the column chunk; dictionaries use `-1`. */
+  pageOrdinal: number;
+  /** Value encoding declared by the page header. */
+  encoding: string;
+  /** Repetition-level encoding for V1 pages. V2 levels always use RLE. */
+  repetitionLevelEncoding?: string;
+  /** Definition-level encoding for V1 pages. V2 levels always use RLE. */
+  definitionLevelEncoding?: string;
+  /** Compression codec declared by the containing column chunk. */
+  compression: string;
+  /** Whether the page value payload remains compressed. */
+  compressionState: ParquetPageCompressionState;
+  /** Number of encoded values declared by the page header. */
+  valueCount: number;
+  /** Number of non-null values when the page header reports it directly. */
+  nonNullValueCount?: number;
+  /** Encoded page body without its Thrift page header. */
+  data: Uint8Array;
+  /** Repetition-level bytes, when their range can be known without decoding values. */
+  repetitionLevels?: ParquetEncodedPageSection;
+  /** Definition-level bytes, when their range can be known without decoding values. */
+  definitionLevels?: ParquetEncodedPageSection;
+  /** Value or dictionary bytes, when their range can be known before deferred decoding. */
+  values?: ParquetEncodedPageSection;
+  /** Compressed body byte length reported by the page header. */
+  compressedByteLength: number;
+  /** Uncompressed body byte length reported by the page header. */
+  uncompressedByteLength: number;
+}>;
+
+/** One projected Parquet leaf column whose encoded pages have not been value-decoded. */
+export type ParquetEncodedColumnChunk = Readonly<{
+  /** Nested Parquet schema path. */
+  path: readonly string[];
+  /** Physical Parquet scalar type. */
+  physicalType: string;
+  /** Fixed byte width for FIXED_LEN_BYTE_ARRAY columns. */
+  typeLength?: number;
+  /** Maximum repetition level for this leaf column. */
+  maxRepetitionLevel: number;
+  /** Maximum definition level for this leaf column. */
+  maxDefinitionLevel: number;
+  /** Compression codec declared by the column chunk. */
+  compression: string;
+  /** Number of encoded values declared by the column chunk. */
+  valueCount: number;
+  /** Optional dictionary page associated only with this column chunk. */
+  dictionary?: ParquetEncodedPage;
+  /** Ordered data pages in this column chunk. */
+  pages: readonly ParquetEncodedPage[];
+}>;
+
+/** Exact filters a deferred decoder must apply after conservative metadata pruning. */
+export type ParquetDeferredPageFilter = Readonly<{
+  /** Exact scalar predicate requested by the caller. */
+  predicate?: ParquetPredicate;
+  /** Exact spatial extent requested by the caller. */
+  bbox?: ParquetBoundingBox;
+  /** Geometry column associated with `bbox`. */
+  geometryColumn?: string;
+}>;
+
+/** Candidate encoded pages returned for one Parquet row group. */
+export type ParquetEncodedPageBatch = Readonly<{
+  /** Stable discriminator for transport-neutral deferred Parquet pages. */
+  shape: 'parquet-encoded-pages';
+  /** Normalized row-group metadata. */
+  rowGroup: ParquetRowGroupMetadata;
+  /** Columns explicitly requested for final output. Empty means all columns. */
+  projectedColumns: readonly string[];
+  /** Hidden columns included so a deferred decoder can evaluate residual filters. */
+  filterColumns: readonly string[];
+  /** Ordered selected column chunks. Each chunk retains its complete nested schema path. */
+  columns: readonly ParquetEncodedColumnChunk[];
+  /** Exact work left after loaders.gl's conservative statistics and Bloom-filter pruning. */
+  residualFilter?: ParquetDeferredPageFilter;
+}>;
+
+/** Options for reading encoded Parquet pages without materializing values or Arrow arrays. */
+export type ParquetEncodedPageReadOptions = Readonly<{
+  /** Zero-based row-group indexes to inspect, in output order. Defaults to all row groups. */
+  rowGroups?: readonly number[];
+  /** Top-level output columns. Defaults to all columns. */
+  columns?: readonly string[];
+  /** Retains candidate row groups for which the callback returns true. */
+  rowGroupFilter?: (rowGroup: ParquetRowGroupMetadata) => boolean;
+  /** Exact predicate used for conservative pruning and returned as residual work. */
+  predicate?: ParquetPredicate;
+  /** Spatial extent used for conservative pruning and returned as residual work. */
+  bbox?: ParquetBoundingBox;
+  /** Geometry column associated with `bbox`; defaults to the GeoParquet primary column. */
+  geometryColumn?: string;
+  /** Compression codecs to preserve for a downstream decoder instead of inflating on the CPU. */
+  preserveCompression?: readonly string[];
+  /** Abort this page read and its outstanding range requests. */
+  signal?: AbortSignal;
+}>;

package/src/parquet-js-loader-types.ts

@@ -21,6 +21,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 const DEFAULT_PARQUET_JS_OPTIONS = {
   columns: undefined,
   preserveBinary: false,
+  verifyFooterSignature: true,
   shape: 'object-row-table' as const
 };
 

package/src/parquet-js-loader.ts

@@ -25,7 +25,8 @@ const {preload: _ParquetJSLoaderPreload, ...ParquetJSLoaderMetadataWithoutPreloa
 /** Default option bag for the experimental parquetjs plain-row loader. */
 const DEFAULT_PARQUET_JS_OPTIONS = {
   columns: undefined,
-  preserveBinary: false
+  preserveBinary: false,
+  verifyFooterSignature: true
 };
 
 /** Parser-bearing TypeScript-only Parquet loader implementation. */

package/src/parquet-js-writer.ts

@@ -9,6 +9,17 @@ import {convertTable, deduceTableSchema} from '@loaders.gl/schema-utils';
 import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
 import {encodeTableToParquetJs} from './lib/encoders/encode-table-to-parquet-js';
 import {ParquetFormat} from './parquet-format';
+import type {ParquetSortingColumnOption} from './parquetjs/encoder/parquet-encoder';
+import type {
+  ParquetWriterEncryptionOptions,
+  ParquetWriterFooterSignatureOptions
+} from './lib/parquet-encryption';
+
+export type {ParquetSortingColumnOption} from './parquetjs/encoder/parquet-encoder';
+export type {
+  ParquetWriterEncryptionOptions,
+  ParquetWriterFooterSignatureOptions
+} from './lib/parquet-encryption';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -17,6 +28,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 /** Stable value encodings selectable by the TypeScript Parquet writer. */
 export type ParquetJSWriterEncoding =
   | 'PLAIN'
+  | 'PLAIN_DICTIONARY'
   | 'BYTE_STREAM_SPLIT'
   | 'DELTA_BINARY_PACKED'
   | 'DELTA_LENGTH_BYTE_ARRAY'
@@ -32,6 +44,24 @@ type ParquetJSWriterEncoderOptions = {
   columnDictionaries?: Record<string, boolean | 'auto'>;
   /** Maximum uncompressed PLAIN dictionary payload per column chunk. */
   dictionaryPageSizeLimit?: number;
+  /** Emits Parquet split-block Bloom filters for selected columns. */
+  bloomFilter?: boolean | Record<string, boolean>;
+  /** Emits Parquet column and offset indexes for selected non-repeated columns. */
+  pageIndex?: boolean | Record<string, boolean>;
+  /** Emits CRC-32 checksums for every data and dictionary page. */
+  writePageChecksums?: boolean;
+  /** Emits optional SizeStatistics metadata for every column chunk. */
+  writeSizeStatistics?: boolean;
+  /** Emits min/max/null-count statistics for every column chunk. */
+  writeStatistics?: boolean | Record<string, boolean>;
+  /** Declares row-group sort keys using top-level or dotted nested leaf names. */
+  sortingColumns?: readonly ParquetSortingColumnOption[];
+  /** Encodes INT96 input values as epoch nanoseconds using the canonical Julian-day layout. */
+  int96AsTimestamp?: boolean;
+  /** Encrypt the footer using Parquet modular encryption. */
+  encryption?: ParquetWriterEncryptionOptions;
+  /** Authenticate a plaintext footer with a Parquet modular-encryption signature. */
+  footerSignature?: ParquetWriterFooterSignatureOptions;
   rowGroupSize?: number;
   pageSize?: number;
   useDataPageV2?: boolean;

package/src/parquet-loader-options.ts

@@ -3,14 +3,19 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import type {ParquetKeyRetriever} from './lib/parquet-encryption';
 
 /** Shared row-level options supported by both Parquet backends. */
 type ParquetCommonLoaderOptions = {
+  /** @deprecated Use `_scan.limit` instead. */
   limit?: number;
   offset?: number;
   batchSize?: number;
+  /** @deprecated Use `_scan.columns` instead for logical projection. */
   columns?: string[];
   preserveBinary?: boolean;
+  /** Decode legacy INT96 values as Arrow timestamp-nanosecond values. */
+  int96AsTimestamp?: boolean;
 };
 
 /** Additional options supported by the wasm-backed Parquet reader. */
@@ -38,6 +43,12 @@ export type ParquetLoaderOptions = LoaderOptions & {
 export type ParquetJSLoaderOptions = LoaderOptions & {
   parquet?: {
     shape?: 'object-row-table' | 'arrow-table';
+    /** Resolves modular-encryption keys from file and column key metadata. */
+    keyRetriever?: ParquetKeyRetriever;
+    /** AAD prefix for encrypted files that intentionally omit it from metadata. */
+    aadPrefix?: Uint8Array;
+    /** Verify plaintext-footer signatures when present. Enabled by default. */
+    verifyFooterSignature?: boolean;
   } & {
     [Key in keyof ParquetCommonLoaderOptions]?: ParquetCommonLoaderOptions[Key];
   };

package/src/parquet-source-capabilities.ts

@@ -2,8 +2,21 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import type {TableQueryCapabilities} from '@loaders.gl/loader-utils';
+
+/** Portable table-query capabilities of Parquet-backed scans. */
+export const PARQUET_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities = Object.freeze({
+  projection: 'pushdown',
+  predicate: 'pushdown+residual',
+  limit: 'pushdown',
+  streaming: true,
+  cancellation: true
+});
+
 /** Feature support advertised by a {@link ParquetSource}. */
 export type ParquetSourceCapabilities = Readonly<{
+  /** Common portable query contract supported by this source. */
+  tableQuery: TableQueryCapabilities;
   /** Schema and footer metadata are cached for the lifetime of the source. */
   supportsCachedMetadata: boolean;
   /** Reads can select explicit Parquet row groups. */
@@ -20,8 +33,10 @@ export type ParquetSourceCapabilities = Readonly<{
   supportsColumnStatistics: boolean;
   /** Serializable predicates can prune impossible row groups using column statistics. */
   supportsPredicatePushdown: boolean;
-  /** Column and offset indexes can avoid irrelevant data-page range reads for flat columns. */
+  /** Column and offset indexes can avoid irrelevant data-page reads for non-repeated columns. */
   supportsPageIndexPruning: boolean;
+  /** Native geospatial statistics and GeoParquet 1.1 coverings drive spatial pruning. */
+  supportsGeoParquetSpatialPruning: boolean;
   /** Surviving decoded rows are filtered exactly on the caller thread or worker. */
   supportsExactPredicateFiltering: boolean;
   /** Callers can supply the random-access transport used by the source. */
@@ -38,6 +53,7 @@ export type ParquetSourceCapabilities = Readonly<{
 
 /** Capabilities of the current range-backed TypeScript {@link ParquetSource}. */
 export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.freeze({
+  tableQuery: PARQUET_TABLE_QUERY_CAPABILITIES,
   supportsCachedMetadata: true,
   supportsRowGroupSelection: true,
   supportsColumnProjection: true,
@@ -47,6 +63,7 @@ export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.fre
   supportsColumnStatistics: true,
   supportsPredicatePushdown: true,
   supportsPageIndexPruning: true,
+  supportsGeoParquetSpatialPruning: true,
   supportsExactPredicateFiltering: true,
   supportsCustomRangeTransport: true,
   supportsObjectVersionValidation: true,

package/src/parquet-source-loader-types.ts

@@ -42,8 +42,13 @@ export const ParquetSourceLoader = {
       batchSize: undefined,
       concurrency: undefined,
       rowGroupFilter: undefined,
+      predicate: undefined,
+      bbox: undefined,
+      geometryColumn: undefined,
       headers: undefined,
       preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
       onTelemetry: undefined,
       workerUrl: undefined,
       wasmUrl: undefined
@@ -61,8 +66,13 @@ export const ParquetSourceLoader = {
       batchSize: undefined!,
       concurrency: undefined!,
       rowGroupFilter: undefined!,
+      predicate: undefined!,
+      bbox: undefined!,
+      geometryColumn: undefined!,
       headers: undefined!,
       preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
       onTelemetry: undefined!,
       workerUrl: undefined!,
       wasmUrl: undefined!