@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/src/parquet-js-writer.ts

@@ -9,6 +9,17 @@ import {convertTable, deduceTableSchema} from '@loaders.gl/schema-utils';
 import {normalizeParquetOptions} from './lib/utils/normalize-parquet-options';
 import {encodeTableToParquetJs} from './lib/encoders/encode-table-to-parquet-js';
 import {ParquetFormat} from './parquet-format';
+import type {ParquetSortingColumnOption} from './parquetjs/encoder/parquet-encoder';
+import type {
+  ParquetWriterEncryptionOptions,
+  ParquetWriterFooterSignatureOptions
+} from './lib/parquet-encryption';
+
+export type {ParquetSortingColumnOption} from './parquetjs/encoder/parquet-encoder';
+export type {
+  ParquetWriterEncryptionOptions,
+  ParquetWriterFooterSignatureOptions
+} from './lib/parquet-encryption';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -17,6 +28,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 /** Stable value encodings selectable by the TypeScript Parquet writer. */
 export type ParquetJSWriterEncoding =
   | 'PLAIN'
+  | 'PLAIN_DICTIONARY'
   | 'BYTE_STREAM_SPLIT'
   | 'DELTA_BINARY_PACKED'
   | 'DELTA_LENGTH_BYTE_ARRAY'
@@ -32,6 +44,24 @@ type ParquetJSWriterEncoderOptions = {
   columnDictionaries?: Record<string, boolean | 'auto'>;
   /** Maximum uncompressed PLAIN dictionary payload per column chunk. */
   dictionaryPageSizeLimit?: number;
+  /** Emits Parquet split-block Bloom filters for selected columns. */
+  bloomFilter?: boolean | Record<string, boolean>;
+  /** Emits Parquet column and offset indexes for selected non-repeated columns. */
+  pageIndex?: boolean | Record<string, boolean>;
+  /** Emits CRC-32 checksums for every data and dictionary page. */
+  writePageChecksums?: boolean;
+  /** Emits optional SizeStatistics metadata for every column chunk. */
+  writeSizeStatistics?: boolean;
+  /** Emits min/max/null-count statistics for every column chunk. */
+  writeStatistics?: boolean | Record<string, boolean>;
+  /** Declares row-group sort keys using top-level or dotted nested leaf names. */
+  sortingColumns?: readonly ParquetSortingColumnOption[];
+  /** Encodes INT96 input values as epoch nanoseconds using the canonical Julian-day layout. */
+  int96AsTimestamp?: boolean;
+  /** Encrypt the footer using Parquet modular encryption. */
+  encryption?: ParquetWriterEncryptionOptions;
+  /** Authenticate a plaintext footer with a Parquet modular-encryption signature. */
+  footerSignature?: ParquetWriterFooterSignatureOptions;
   rowGroupSize?: number;
   pageSize?: number;
   useDataPageV2?: boolean;

package/src/parquet-loader-options.ts

@@ -3,14 +3,19 @@
 // Copyright (c) vis.gl contributors
 
 import type {LoaderOptions} from '@loaders.gl/loader-utils';
+import type {ParquetKeyRetriever} from './lib/parquet-encryption';
 
 /** Shared row-level options supported by both Parquet backends. */
 type ParquetCommonLoaderOptions = {
+  /** @deprecated Use `_scan.limit` instead. */
   limit?: number;
   offset?: number;
   batchSize?: number;
+  /** @deprecated Use `_scan.columns` instead for logical projection. */
   columns?: string[];
   preserveBinary?: boolean;
+  /** Decode legacy INT96 values as Arrow timestamp-nanosecond values. */
+  int96AsTimestamp?: boolean;
 };
 
 /** Additional options supported by the wasm-backed Parquet reader. */
@@ -38,6 +43,12 @@ export type ParquetLoaderOptions = LoaderOptions & {
 export type ParquetJSLoaderOptions = LoaderOptions & {
   parquet?: {
     shape?: 'object-row-table' | 'arrow-table';
+    /** Resolves modular-encryption keys from file and column key metadata. */
+    keyRetriever?: ParquetKeyRetriever;
+    /** AAD prefix for encrypted files that intentionally omit it from metadata. */
+    aadPrefix?: Uint8Array;
+    /** Verify plaintext-footer signatures when present. Enabled by default. */
+    verifyFooterSignature?: boolean;
   } & {
     [Key in keyof ParquetCommonLoaderOptions]?: ParquetCommonLoaderOptions[Key];
   };

package/src/parquet-source-capabilities.ts

@@ -2,8 +2,21 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import type {TableQueryCapabilities} from '@loaders.gl/loader-utils';
+
+/** Portable table-query capabilities of Parquet-backed scans. */
+export const PARQUET_TABLE_QUERY_CAPABILITIES: TableQueryCapabilities = Object.freeze({
+  projection: 'pushdown',
+  predicate: 'pushdown+residual',
+  limit: 'pushdown',
+  streaming: true,
+  cancellation: true
+});
+
 /** Feature support advertised by a {@link ParquetSource}. */
 export type ParquetSourceCapabilities = Readonly<{
+  /** Common portable query contract supported by this source. */
+  tableQuery: TableQueryCapabilities;
   /** Schema and footer metadata are cached for the lifetime of the source. */
   supportsCachedMetadata: boolean;
   /** Reads can select explicit Parquet row groups. */
@@ -20,8 +33,10 @@ export type ParquetSourceCapabilities = Readonly<{
   supportsColumnStatistics: boolean;
   /** Serializable predicates can prune impossible row groups using column statistics. */
   supportsPredicatePushdown: boolean;
-  /** Column and offset indexes can avoid irrelevant data-page range reads for flat columns. */
+  /** Column and offset indexes can avoid irrelevant data-page reads for non-repeated columns. */
   supportsPageIndexPruning: boolean;
+  /** Native geospatial statistics and GeoParquet 1.1 coverings drive spatial pruning. */
+  supportsGeoParquetSpatialPruning: boolean;
   /** Surviving decoded rows are filtered exactly on the caller thread or worker. */
   supportsExactPredicateFiltering: boolean;
   /** Callers can supply the random-access transport used by the source. */
@@ -38,6 +53,7 @@ export type ParquetSourceCapabilities = Readonly<{
 
 /** Capabilities of the current range-backed TypeScript {@link ParquetSource}. */
 export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.freeze({
+  tableQuery: PARQUET_TABLE_QUERY_CAPABILITIES,
   supportsCachedMetadata: true,
   supportsRowGroupSelection: true,
   supportsColumnProjection: true,
@@ -47,6 +63,7 @@ export const PARQUET_SOURCE_CAPABILITIES: ParquetSourceCapabilities = Object.fre
   supportsColumnStatistics: true,
   supportsPredicatePushdown: true,
   supportsPageIndexPruning: true,
+  supportsGeoParquetSpatialPruning: true,
   supportsExactPredicateFiltering: true,
   supportsCustomRangeTransport: true,
   supportsObjectVersionValidation: true,

package/src/parquet-source-loader-types.ts

@@ -42,8 +42,13 @@ export const ParquetSourceLoader = {
       batchSize: undefined,
       concurrency: undefined,
       rowGroupFilter: undefined,
+      predicate: undefined,
+      bbox: undefined,
+      geometryColumn: undefined,
       headers: undefined,
       preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
       onTelemetry: undefined,
       workerUrl: undefined,
       wasmUrl: undefined
@@ -61,8 +66,13 @@ export const ParquetSourceLoader = {
       batchSize: undefined!,
       concurrency: undefined!,
       rowGroupFilter: undefined!,
+      predicate: undefined!,
+      bbox: undefined!,
+      geometryColumn: undefined!,
       headers: undefined!,
       preserveBinary: false,
+      verifyPageChecksums: false,
+      verifyFooterSignature: true,
       onTelemetry: undefined!,
       workerUrl: undefined!,
       wasmUrl: undefined!