@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.4

package/dist/parquet-source-loader.d.ts

@@ -1,10 +1,13 @@
 import type { CoreAPI } from '@loaders.gl/loader-utils';
 import { DataSource } from '@loaders.gl/loader-utils';
+import type { ScanQueryMetadata, ScanQueryMetadataOptions, TableScanSource } from '@loaders.gl/loader-utils';
 import type { Schema } from '@loaders.gl/schema';
 import { type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
-import type { ParquetMetadataRequestOptions, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetTelemetry } from "./parquet-source-types.js";
-export { PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
-export type { ParquetBatch, ParquetBatchMetadata, ParquetBatchProvenance, ParquetColumnChunkMetadata, ParquetColumnChunkStatistics, ParquetMetadataRequestOptions, ParquetObjectVersion, ParquetComparisonPredicate, ParquetInPredicate, ParquetLogicalPredicate, ParquetNotPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetPredicateValue, ParquetRangeRequestOptions, ParquetReadOptions, ParquetRowGroupMetadata, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetTelemetry, ParquetTelemetryEvent } from "./parquet-source-types.js";
+import type { ParquetEncodedPageBatch, ParquetEncodedPageReadOptions } from "./parquet-encoded-page-types.js";
+import type { ParquetMetadataRequestOptions, ParquetPredicate, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetSourceExplain, ParquetTelemetry } from "./parquet-source-types.js";
+export { PARQUET_TABLE_QUERY_CAPABILITIES, PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
+export type { ParquetEncodedColumnChunk, ParquetEncodedPage, ParquetEncodedPageBatch, ParquetEncodedPageReadOptions, ParquetEncodedPageSection, ParquetDeferredPageFilter, ParquetPageCompressionState } from "./parquet-encoded-page-types.js";
+export type { ParquetBatch, ParquetBatchMetadata, ParquetBatchProvenance, ParquetBoundingBox, ParquetColumnChunkMetadata, ParquetColumnChunkStatistics, ParquetColumnChunkSizeStatistics, ParquetGeospatialBoundingBox, ParquetGeospatialStatistics, ParquetMetadataRequestOptions, ParquetObjectVersion, ParquetPageScanPlan, ParquetComparisonPredicate, ParquetInPredicate, ParquetLogicalPredicate, ParquetNotPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetPredicateValue, ParquetRangeRequestOptions, ParquetReadOptions, ParquetRowGroupMetadata, ParquetSortingColumn, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetSourceExplain, ParquetTelemetry, ParquetTelemetryEvent } from "./parquet-source-types.js";
 /** Runtime source factory for reusable, range-backed Parquet access. */
 export declare const ParquetSourceLoaderWithParser: {
     readonly createDataSource: (data: string | Blob, options: ParquetSourceLoaderOptions, coreApi?: CoreAPI) => ParquetSource;
@@ -24,8 +27,13 @@ export declare const ParquetSourceLoaderWithParser: {
             readonly batchSize: undefined;
             readonly concurrency: undefined;
             readonly rowGroupFilter: undefined;
+            readonly predicate: undefined;
+            readonly bbox: undefined;
+            readonly geometryColumn: undefined;
             readonly headers: undefined;
             readonly preserveBinary: false;
+            readonly verifyPageChecksums: false;
+            readonly verifyFooterSignature: true;
             readonly onTelemetry: undefined;
             readonly workerUrl: undefined;
             readonly wasmUrl: undefined;
@@ -43,8 +51,13 @@ export declare const ParquetSourceLoaderWithParser: {
             readonly batchSize: never;
             readonly concurrency: never;
             readonly rowGroupFilter: never;
+            readonly predicate: never;
+            readonly bbox: never;
+            readonly geometryColumn: never;
             readonly headers: never;
             readonly preserveBinary: false;
+            readonly verifyPageChecksums: false;
+            readonly verifyFooterSignature: true;
             readonly onTelemetry: never;
             readonly workerUrl: never;
             readonly wasmUrl: never;
@@ -67,7 +80,18 @@ export declare const ParquetSourceLoaderWithParser: {
 /** Runtime Parquet source loader exposed by the explicit package subpath. */
 export { ParquetSourceLoaderWithParser as ParquetSourceLoader };
 /** Reusable Parquet source that caches footer/schema state and selectively reads byte ranges. */
-export declare class ParquetSource extends DataSource<string | Blob, ParquetSourceLoaderOptions> {
+export declare class ParquetSource extends DataSource<string | Blob, ParquetSourceLoaderOptions> implements TableScanSource<ParquetSourceBatch, ParquetPredicate> {
+    /** Common projection, predicate, limit, streaming, and cancellation capabilities. */
+    readonly tableQueryCapabilities: Readonly<{
+        projection: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        predicate: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        limit: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        streaming: boolean;
+        cancellation: boolean;
+        expressions?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        orderBy?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        aggregates?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+    }>;
     /** Immutable feature support for the current range-backed source implementation. */
     readonly capabilities: ParquetSourceCapabilities;
     /** Shared initialization for this source instance. */
@@ -90,10 +114,31 @@ export declare class ParquetSource extends DataSource<string | Blob, ParquetSour
     }): Promise<Schema>;
     /** Returns normalized dataset, row-group, and column-chunk metadata. */
     getMetadata(options?: ParquetMetadataRequestOptions): Promise<ParquetSourceMetadata>;
+    /** Discovers schema, capabilities, and footer statistics without decoding Parquet data pages. */
+    getQueryMetadata(options?: ScanQueryMetadataOptions): Promise<ScanQueryMetadata>;
+    /** Plans a portable query using Parquet metadata without decoding data pages. */
+    getScanPlan(options?: ParquetSourceReadOptions): Promise<ParquetSourceExplain>;
+    /** Explains the common logical query and Parquet-specific physical scan plan. */
+    explain(options?: ParquetSourceReadOptions): Promise<ParquetSourceExplain>;
+    /** Executes a previously computed physical plan while preserving its row-group decisions. */
+    executeScanPlan(plan: ParquetSourceExplain, options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /** Common scan-architecture alias for selective Parquet reads. */
+    scan(options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /**
+     * Reads candidate encoded pages without decoding values or constructing Arrow arrays.
+     *
+     * Footer statistics and Bloom filters prune complete row groups. Any exact predicate or spatial
+     * filter is returned as residual work, and its hidden columns are included in the page batch.
+     */
+    readPages(options?: ParquetEncodedPageReadOptions): AsyncIterable<ParquetEncodedPageBatch>;
     /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
     getTelemetry(): ParquetTelemetry;
     /** Selectively fetches row groups and columns as ordered Arrow batches with source provenance. */
     read(options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /** Produces the physical row-group plan shared by explain and execution. */
+    private planRowGroups;
+    /** Produces explainable page-index and byte-range plans for retained row groups. */
+    private planPages;
     /** Closes the underlying readable file and aborts active remote requests. */
     close(): Promise<void>;
     /** Returns cached initialization, resetting the cache when initialization fails. */
@@ -108,6 +153,8 @@ export declare class ParquetSource extends DataSource<string | Blob, ParquetSour
     private readRowGroup;
     /** Fetches selected chunks and transfers their decompression and Arrow conversion to a worker. */
     private readRowGroupOnWorker;
+    /** Decodes predicate columns first, avoiding projected-column decoding for empty matches. */
+    private readRowGroupWithLateMaterialization;
     /** Records the rows and pages avoided by one conservative page-index plan. */
     private recordPagePruningTelemetry;
     /** Opens the readable file and decodes its footer and schema once. */

package/dist/parquet-source-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-source-loader.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,OAAO,EAA6B,MAAM,0BAA0B,CAAC;AAClF,OAAO,EAAW,UAAU,EAAY,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAwB,MAAM,EAAC,MAAM,oBAAoB,CAAC;AA2BtE,OAAO,EAEL,KAAK,yBAAyB,EAC/B,yCAAsC;AAEvC,OAAO,KAAK,EAIV,6BAA6B,EAI7B,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAEjB,kCAA+B;AAChC,OAAO,EACL,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC;AAyBvC,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,0BAA0B,EAC1B,4BAA4B,EAC5B,6BAA6B,EAC7B,oBAAoB,EACpB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,qBAAqB,EACtB,kCAA+B;AAkDhC,wEAAwE;AACxE,eAAO,MAAM,6BAA6B;sCAGhC,MAAM,GAAG,IAAI,WACV,0BAA0B,YACzB,OAAO,KAChB,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAC8B,CAAC;AAEjD,6EAA6E;AAC7E,OAAO,EAAC,6BAA6B,IAAI,mBAAmB,EAAC,CAAC;AAE9D,iGAAiG;AACjG,qBAAa,aAAc,SAAQ,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,0BAA0B,CAAC;IACtF,oFAAoF;IACpF,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAA+B;IAC/E,sDAAsD;IACtD,OAAO,CAAC,qBAAqB,CAAqD;IAClF,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IACjD,uDAAuD;IACvD,OAAO,CAAC,MAAM,CAAS;IACvB,gEAAgE;IAChE,OAAO,CAAC,kBAAkB,CAA8B;IACxD,qFAAqF;IACrF,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,4CAA4C;IAC5C,OAAO,CAAC,SAAS,CAA4B;IAE7C,2EAA2E;gBAC/D,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,0BAA0B,EAAE,OAAO,CAAC,EAAE,OAAO;IAQvF,yEAAyE;IACnE,SAAS,CAAC,OAAO,GAAE;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKtE,wEAAwE;IAClE,WAAW,CAAC,OAAO,GAAE,6BAAkC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAW9F,yFAAyF;IACzF,YAAY,IAAI,gBAAgB;IAIhC,kGAAkG;IAC3F,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAgMtF,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAe5B,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAazB,4FAA4F;IAC5F,OAAO,CAAC,4BAA4B;IAYpC,6FAA6F;IAC7F,OAAO,CAAC,cAAc;IAetB,yEAAyE;IACzE,OAAO,CAAC,gBAAgB;IAkBxB,qFAAqF;YACvE,YAAY;IA4G1B,kGAAkG;YACpF,oBAAoB;IAwElC,8EAA8E;IAC9E,OAAO,CAAC,0BAA0B;IAelC,sEAAsE;YACxD,UAAU;IA2BxB,kFAAkF;YACpE,QAAQ;IA4BtB,4FAA4F;IAC5F,OAAO,CAAC,eAAe;CAkBxB"}
+{"version":3,"file":"parquet-source-loader.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,OAAO,EAA6B,MAAM,0BAA0B,CAAC;AAClF,OAAO,EAEL,UAAU,EAOX,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAGV,iBAAiB,EACjB,wBAAwB,EACxB,eAAe,EAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAwB,MAAM,EAAC,MAAM,oBAAoB,CAAC;AA2CtE,OAAO,EAGL,KAAK,yBAAyB,EAC/B,yCAAsC;AAEvC,OAAO,KAAK,EACV,uBAAuB,EACvB,6BAA6B,EAE9B,wCAAqC;AACtC,OAAO,KAAK,EAMV,6BAA6B,EAG7B,gBAAgB,EAGhB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAEjB,kCAA+B;AAChC,OAAO,EACL,gCAAgC,EAChC,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC;AA2BvC,YAAY,EACV,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,6BAA6B,EAC7B,yBAAyB,EACzB,yBAAyB,EACzB,2BAA2B,EAC5B,wCAAqC;AACtC,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,0BAA0B,EAC1B,4BAA4B,EAC5B,gCAAgC,EAChC,4BAA4B,EAC5B,2BAA2B,EAC3B,6BAA6B,EAC7B,oBAAoB,EACpB,mBAAmB,EACnB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,oBAAoB,EACpB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACtB,kCAA+B;AAuEhC,wEAAwE;AACxE,eAAO,MAAM,6BAA6B;sCAGhC,MAAM,GAAG,IAAI,WACV,0BAA0B,YACzB,OAAO,KAChB,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAC8B,CAAC;AAEjD,6EAA6E;AAC7E,OAAO,EAAC,6BAA6B,IAAI,mBAAmB,EAAC,CAAC;AAE9D,iGAAiG;AACjG,qBAAa,aACX,SAAQ,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,0BAA0B,CAC5D,YAAW,eAAe,CAAC,kBAAkB,EAAE,gBAAgB,CAAC;IAEhE,qFAAqF;IACrF,QAAQ,CAAC,sBAAsB;;;;;;;;;OAAoC;IACnE,oFAAoF;IACpF,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAA+B;IAC/E,sDAAsD;IACtD,OAAO,CAAC,qBAAqB,CAAqD;IAClF,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IACjD,uDAAuD;IACvD,OAAO,CAAC,MAAM,CAAS;IACvB,gEAAgE;IAChE,OAAO,CAAC,kBAAkB,CAA8B;IACxD,qFAAqF;IACrF,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,4CAA4C;IAC5C,OAAO,CAAC,SAAS,CAA4B;IAE7C,2EAA2E;gBAC/D,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,0BAA0B,EAAE,OAAO,CAAC,EAAE,OAAO;IAQvF,yEAAyE;IACnE,SAAS,CAAC,OAAO,GAAE;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKtE,wEAAwE;IAClE,WAAW,CAAC,OAAO,GAAE,6BAAkC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAW9F,iGAAiG;IAC3F,gBAAgB,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAiB1F,iFAAiF;IAC3E,WAAW,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA2CxF,iFAAiF;IAC3E,OAAO,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAIpF,6FAA6F;IACtF,eAAe,CACpB,IAAI,EAAE,oBAAoB,EAC1B,OAAO,GAAE,wBAA6B,GACrC,aAAa,CAAC,kBAAkB,CAAC;IAiBpC,kEAAkE;IAClE,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAI/E;;;;;OAKG;IACI,SAAS,CACd,OAAO,GAAE,6BAAkC,GAC1C,aAAa,CAAC,uBAAuB,CAAC;IAwEzC,yFAAyF;IACzF,YAAY,IAAI,gBAAgB;IAIhC,kGAAkG;IAC3F,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAiPtF,4EAA4E;YAC9D,aAAa;IAoE3B,oFAAoF;YACtE,SAAS;IAiGvB,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAazB,4FAA4F;IAC5F,OAAO,CAAC,4BAA4B;IAYpC,6FAA6F;IAC7F,OAAO,CAAC,cAAc;IAqBtB,yEAAyE;IACzE,OAAO,CAAC,gBAAgB;IAkBxB,qFAAqF;YACvE,YAAY;IA4I1B,kGAAkG;YACpF,oBAAoB;IAqGlC,6FAA6F;YAC/E,mCAAmC;IAiEjD,8EAA8E;IAC9E,OAAO,CAAC,0BAA0B;IAelC,sEAAsE;YACxD,UAAU;IAgCxB,kFAAkF;YACpE,QAAQ;IA4BtB,4FAA4F;IAC5F,OAAO,CAAC,eAAe;CAkBxB"}