@loaders.gl/parquet 4.5.3 → 5.0.0-alpha.10

package/dist/parquet-source-loader.d.ts

@@ -0,0 +1,167 @@
+import type { CoreAPI } from '@loaders.gl/loader-utils';
+import { DataSource } from '@loaders.gl/loader-utils';
+import type { ScanQueryMetadata, ScanQueryMetadataOptions, TableScanSource } from '@loaders.gl/loader-utils';
+import type { Schema } from '@loaders.gl/schema';
+import { type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
+import type { ParquetEncodedPageBatch, ParquetEncodedPageReadOptions } from "./parquet-encoded-page-types.js";
+import type { ParquetMetadataRequestOptions, ParquetPredicate, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetSourceExplain, ParquetTelemetry } from "./parquet-source-types.js";
+export { PARQUET_TABLE_QUERY_CAPABILITIES, PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
+export type { ParquetEncodedColumnChunk, ParquetEncodedPage, ParquetEncodedPageBatch, ParquetEncodedPageReadOptions, ParquetEncodedPageSection, ParquetDeferredPageFilter, ParquetPageCompressionState } from "./parquet-encoded-page-types.js";
+export type { ParquetBatch, ParquetBatchMetadata, ParquetBatchProvenance, ParquetBoundingBox, ParquetColumnChunkMetadata, ParquetColumnChunkStatistics, ParquetColumnChunkSizeStatistics, ParquetGeospatialBoundingBox, ParquetGeospatialStatistics, ParquetMetadataRequestOptions, ParquetObjectVersion, ParquetPageScanPlan, ParquetComparisonPredicate, ParquetInPredicate, ParquetLogicalPredicate, ParquetNotPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetPredicateValue, ParquetRangeRequestOptions, ParquetReadOptions, ParquetRowGroupMetadata, ParquetSortingColumn, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetSourceExplain, ParquetTelemetry, ParquetTelemetryEvent } from "./parquet-source-types.js";
+/** Runtime source factory for reusable, range-backed Parquet access. */
+export declare const ParquetSourceLoaderWithParser: {
+    readonly createDataSource: (data: string | Blob, options: ParquetSourceLoaderOptions, coreApi?: CoreAPI) => ParquetSource;
+    readonly dataType: ParquetSource;
+    readonly batchType: never;
+    readonly name: "ParquetSourceLoader";
+    readonly id: "parquet-source";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly type: "parquet";
+    readonly fromUrl: true;
+    readonly fromBlob: true;
+    readonly options: {
+        readonly parquet: {
+            readonly rowGroups: undefined;
+            readonly columns: undefined;
+            readonly batchSize: undefined;
+            readonly concurrency: undefined;
+            readonly rowGroupFilter: undefined;
+            readonly predicate: undefined;
+            readonly bbox: undefined;
+            readonly geometryColumn: undefined;
+            readonly headers: undefined;
+            readonly preserveBinary: false;
+            readonly verifyPageChecksums: false;
+            readonly verifyFooterSignature: true;
+            readonly onTelemetry: undefined;
+            readonly workerUrl: undefined;
+            readonly wasmUrl: undefined;
+        };
+        readonly rangeRequests: {
+            readonly batchDelayMs: 0;
+            readonly maxGapBytes: 0;
+            readonly rangeExpansionBytes: 0;
+        };
+    };
+    readonly defaultOptions: {
+        readonly parquet: {
+            readonly rowGroups: never;
+            readonly columns: never;
+            readonly batchSize: never;
+            readonly concurrency: never;
+            readonly rowGroupFilter: never;
+            readonly predicate: never;
+            readonly bbox: never;
+            readonly geometryColumn: never;
+            readonly headers: never;
+            readonly preserveBinary: false;
+            readonly verifyPageChecksums: false;
+            readonly verifyFooterSignature: true;
+            readonly onTelemetry: never;
+            readonly workerUrl: never;
+            readonly wasmUrl: never;
+        };
+        readonly rangeRequests: {
+            readonly batchDelayMs: 0;
+            readonly maxGapBytes: 0;
+            readonly rangeExpansionBytes: 0;
+        };
+    };
+    readonly testURL: (url: string) => boolean;
+    readonly category: "table";
+    readonly encoding: "parquet";
+    readonly format: "parquet";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+};
+/** Runtime Parquet source loader exposed by the explicit package subpath. */
+export { ParquetSourceLoaderWithParser as ParquetSourceLoader };
+/** Reusable Parquet source that caches footer/schema state and selectively reads byte ranges. */
+export declare class ParquetSource extends DataSource<string | Blob, ParquetSourceLoaderOptions> implements TableScanSource<ParquetSourceBatch, ParquetPredicate> {
+    /** Common projection, predicate, limit, streaming, and cancellation capabilities. */
+    readonly tableQueryCapabilities: Readonly<{
+        projection: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        predicate: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        limit: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        streaming: boolean;
+        cancellation: boolean;
+        expressions?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        orderBy?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+        aggregates?: import("@loaders.gl/loader-utils").TableQueryOperatorSupport;
+    }>;
+    /** Immutable feature support for the current range-backed source implementation. */
+    readonly capabilities: ParquetSourceCapabilities;
+    /** Shared initialization for this source instance. */
+    private initializationPromise;
+    /** File allocated during source initialization, including while it is opening. */
+    private readableFile;
+    /** Whether this source has been permanently closed. */
+    private closed;
+    /** Compression module preload shared by all selective reads. */
+    private compressionPromise;
+    /** Read abort controllers used to cancel active iterators when the source closes. */
+    private activeReadControllers;
+    /** Cumulative source telemetry counters. */
+    private telemetry;
+    /** Creates a Parquet source backed by strict URL ranges or Blob slices. */
+    constructor(data: string | Blob, options: ParquetSourceLoaderOptions, coreApi?: CoreAPI);
+    /** Returns the cached logical schema decoded from the Parquet footer. */
+    getSchema(options?: {
+        signal?: AbortSignal;
+    }): Promise<Schema>;
+    /** Returns normalized dataset, row-group, and column-chunk metadata. */
+    getMetadata(options?: ParquetMetadataRequestOptions): Promise<ParquetSourceMetadata>;
+    /** Discovers schema, capabilities, and footer statistics without decoding Parquet data pages. */
+    getQueryMetadata(options?: ScanQueryMetadataOptions): Promise<ScanQueryMetadata>;
+    /** Plans a portable query using Parquet metadata without decoding data pages. */
+    getScanPlan(options?: ParquetSourceReadOptions): Promise<ParquetSourceExplain>;
+    /** Explains the common logical query and Parquet-specific physical scan plan. */
+    explain(options?: ParquetSourceReadOptions): Promise<ParquetSourceExplain>;
+    /** Executes a previously computed physical plan while preserving its row-group decisions. */
+    executeScanPlan(plan: ParquetSourceExplain, options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /** Common scan-architecture alias for selective Parquet reads. */
+    scan(options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /**
+     * Reads candidate encoded pages without decoding values or constructing Arrow arrays.
+     *
+     * Footer statistics and Bloom filters prune complete row groups. Any exact predicate or spatial
+     * filter is returned as residual work, and its hidden columns are included in the page batch.
+     */
+    readPages(options?: ParquetEncodedPageReadOptions): AsyncIterable<ParquetEncodedPageBatch>;
+    /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
+    getTelemetry(): ParquetTelemetry;
+    /** Selectively fetches row groups and columns as ordered Arrow batches with source provenance. */
+    read(options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /** Produces the physical row-group plan shared by explain and execution. */
+    private planRowGroups;
+    /** Produces explainable page-index and byte-range plans for retained row groups. */
+    private planPages;
+    /** Closes the underlying readable file and aborts active remote requests. */
+    close(): Promise<void>;
+    /** Returns cached initialization, resetting the cache when initialization fails. */
+    private getInitialization;
+    /** Preloads TypeScript decoder compression modules once and permits retry after failure. */
+    private getCompressionInitialization;
+    /** Snapshots source defaults and per-read overrides before asynchronous iteration begins. */
+    private getReadOptions;
+    /** Creates loader-utils worker options for one selective source read. */
+    private getWorkerOptions;
+    /** Fetches and decodes one selected row group using the cached footer and schema. */
+    private readRowGroup;
+    /** Fetches selected chunks and transfers their decompression and Arrow conversion to a worker. */
+    private readRowGroupOnWorker;
+    /** Decodes predicate columns first, avoiding projected-column decoding for empty matches. */
+    private readRowGroupWithLateMaterialization;
+    /** Records the rows and pages avoided by one conservative page-index plan. */
+    private recordPagePruningTelemetry;
+    /** Opens the readable file and decodes its footer and schema once. */
+    private initialize;
+    /** Opens a Blob file or probes a remote object with one bounded range request. */
+    private openFile;
+    /** Applies telemetry deltas and emits a callback with the resulting cumulative snapshot. */
+    private recordTelemetry;
+}
+//# sourceMappingURL=parquet-source-loader.d.ts.map

package/dist/parquet-source-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-loader.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,OAAO,EAA6B,MAAM,0BAA0B,CAAC;AAClF,OAAO,EAEL,UAAU,EAOX,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAGV,iBAAiB,EACjB,wBAAwB,EACxB,eAAe,EAChB,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAwB,MAAM,EAAC,MAAM,oBAAoB,CAAC;AA2CtE,OAAO,EAGL,KAAK,yBAAyB,EAC/B,yCAAsC;AAEvC,OAAO,KAAK,EACV,uBAAuB,EACvB,6BAA6B,EAE9B,wCAAqC;AACtC,OAAO,KAAK,EAMV,6BAA6B,EAG7B,gBAAgB,EAGhB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAEjB,kCAA+B;AAChC,OAAO,EACL,gCAAgC,EAChC,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC;AA2BvC,YAAY,EACV,yBAAyB,EACzB,kBAAkB,EAClB,uBAAuB,EACvB,6BAA6B,EAC7B,yBAAyB,EACzB,yBAAyB,EACzB,2BAA2B,EAC5B,wCAAqC;AACtC,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,kBAAkB,EAClB,0BAA0B,EAC1B,4BAA4B,EAC5B,gCAAgC,EAChC,4BAA4B,EAC5B,2BAA2B,EAC3B,6BAA6B,EAC7B,oBAAoB,EACpB,mBAAmB,EACnB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,oBAAoB,EACpB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,oBAAoB,EACpB,gBAAgB,EAChB,qBAAqB,EACtB,kCAA+B;AAuEhC,wEAAwE;AACxE,eAAO,MAAM,6BAA6B;sCAGhC,MAAM,GAAG,IAAI,WACV,0BAA0B,YACzB,OAAO,KAChB,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAC8B,CAAC;AAEjD,6EAA6E;AAC7E,OAAO,EAAC,6BAA6B,IAAI,mBAAmB,EAAC,CAAC;AAE9D,iGAAiG;AACjG,qBAAa,aACX,SAAQ,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,0BAA0B,CAC5D,YAAW,eAAe,CAAC,kBAAkB,EAAE,gBAAgB,CAAC;IAEhE,qFAAqF;IACrF,QAAQ,CAAC,sBAAsB;;;;;;;;;OAAoC;IACnE,oFAAoF;IACpF,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAA+B;IAC/E,sDAAsD;IACtD,OAAO,CAAC,qBAAqB,CAAqD;IAClF,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IACjD,uDAAuD;IACvD,OAAO,CAAC,MAAM,CAAS;IACvB,gEAAgE;IAChE,OAAO,CAAC,kBAAkB,CAA8B;IACxD,qFAAqF;IACrF,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,4CAA4C;IAC5C,OAAO,CAAC,SAAS,CAA4B;IAE7C,2EAA2E;gBAC/D,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,0BAA0B,EAAE,OAAO,CAAC,EAAE,OAAO;IAQvF,yEAAyE;IACnE,SAAS,CAAC,OAAO,GAAE;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKtE,wEAAwE;IAClE,WAAW,CAAC,OAAO,GAAE,6BAAkC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAW9F,iGAAiG;IAC3F,gBAAgB,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,iBAAiB,CAAC;IAiB1F,iFAAiF;IAC3E,WAAW,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IA2CxF,iFAAiF;IAC3E,OAAO,CAAC,OAAO,GAAE,wBAA6B,GAAG,OAAO,CAAC,oBAAoB,CAAC;IAIpF,6FAA6F;IACtF,eAAe,CACpB,IAAI,EAAE,oBAAoB,EAC1B,OAAO,GAAE,wBAA6B,GACrC,aAAa,CAAC,kBAAkB,CAAC;IAiBpC,kEAAkE;IAClE,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAI/E;;;;;OAKG;IACI,SAAS,CACd,OAAO,GAAE,6BAAkC,GAC1C,aAAa,CAAC,uBAAuB,CAAC;IAwEzC,yFAAyF;IACzF,YAAY,IAAI,gBAAgB;IAIhC,kGAAkG;IAC3F,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAiPtF,4EAA4E;YAC9D,aAAa;IAoE3B,oFAAoF;YACtE,SAAS;IAiGvB,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAgB5B,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAazB,4FAA4F;IAC5F,OAAO,CAAC,4BAA4B;IAYpC,6FAA6F;IAC7F,OAAO,CAAC,cAAc;IAqBtB,yEAAyE;IACzE,OAAO,CAAC,gBAAgB;IAkBxB,qFAAqF;YACvE,YAAY;IA4I1B,kGAAkG;YACpF,oBAAoB;IAqGlC,6FAA6F;YAC/E,mCAAmC;IAiEjD,8EAA8E;IAC9E,OAAO,CAAC,0BAA0B;IAelC,sEAAsE;YACxD,UAAU;IAgCxB,kFAAkF;YACpE,QAAQ;IA4BtB,4FAA4F;IAC5F,OAAO,CAAC,eAAe;CAkBxB"}