@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/dist/parquet-source-loader.d.ts

@@ -0,0 +1,120 @@
+import type { CoreAPI } from '@loaders.gl/loader-utils';
+import { DataSource } from '@loaders.gl/loader-utils';
+import type { Schema } from '@loaders.gl/schema';
+import { type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
+import type { ParquetMetadataRequestOptions, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetTelemetry } from "./parquet-source-types.js";
+export { PARQUET_SOURCE_CAPABILITIES, type ParquetSourceCapabilities } from "./parquet-source-capabilities.js";
+export type { ParquetBatch, ParquetBatchMetadata, ParquetBatchProvenance, ParquetColumnChunkMetadata, ParquetColumnChunkStatistics, ParquetMetadataRequestOptions, ParquetObjectVersion, ParquetComparisonPredicate, ParquetInPredicate, ParquetLogicalPredicate, ParquetNotPredicate, ParquetNullPredicate, ParquetPredicate, ParquetPredicateProperty, ParquetPredicateValue, ParquetRangeRequestOptions, ParquetReadOptions, ParquetRowGroupMetadata, ParquetSourceBatch, ParquetSourceLoaderOptions, ParquetSourceMetadata, ParquetSourceReadOptions, ParquetTelemetry, ParquetTelemetryEvent } from "./parquet-source-types.js";
+/** Runtime source factory for reusable, range-backed Parquet access. */
+export declare const ParquetSourceLoaderWithParser: {
+    readonly createDataSource: (data: string | Blob, options: ParquetSourceLoaderOptions, coreApi?: CoreAPI) => ParquetSource;
+    readonly dataType: ParquetSource;
+    readonly batchType: never;
+    readonly name: "ParquetSourceLoader";
+    readonly id: "parquet-source";
+    readonly module: "parquet";
+    readonly version: any;
+    readonly type: "parquet";
+    readonly fromUrl: true;
+    readonly fromBlob: true;
+    readonly options: {
+        readonly parquet: {
+            readonly rowGroups: undefined;
+            readonly columns: undefined;
+            readonly batchSize: undefined;
+            readonly concurrency: undefined;
+            readonly rowGroupFilter: undefined;
+            readonly headers: undefined;
+            readonly preserveBinary: false;
+            readonly onTelemetry: undefined;
+            readonly workerUrl: undefined;
+            readonly wasmUrl: undefined;
+        };
+        readonly rangeRequests: {
+            readonly batchDelayMs: 0;
+            readonly maxGapBytes: 0;
+            readonly rangeExpansionBytes: 0;
+        };
+    };
+    readonly defaultOptions: {
+        readonly parquet: {
+            readonly rowGroups: never;
+            readonly columns: never;
+            readonly batchSize: never;
+            readonly concurrency: never;
+            readonly rowGroupFilter: never;
+            readonly headers: never;
+            readonly preserveBinary: false;
+            readonly onTelemetry: never;
+            readonly workerUrl: never;
+            readonly wasmUrl: never;
+        };
+        readonly rangeRequests: {
+            readonly batchDelayMs: 0;
+            readonly maxGapBytes: 0;
+            readonly rangeExpansionBytes: 0;
+        };
+    };
+    readonly testURL: (url: string) => boolean;
+    readonly category: "table";
+    readonly encoding: "parquet";
+    readonly format: "parquet";
+    readonly extensions: ["parquet"];
+    readonly mimeTypes: ["application/octet-stream"];
+    readonly binary: true;
+    readonly tests: ["PAR1", "PARE"];
+};
+/** Runtime Parquet source loader exposed by the explicit package subpath. */
+export { ParquetSourceLoaderWithParser as ParquetSourceLoader };
+/** Reusable Parquet source that caches footer/schema state and selectively reads byte ranges. */
+export declare class ParquetSource extends DataSource<string | Blob, ParquetSourceLoaderOptions> {
+    /** Immutable feature support for the current range-backed source implementation. */
+    readonly capabilities: ParquetSourceCapabilities;
+    /** Shared initialization for this source instance. */
+    private initializationPromise;
+    /** File allocated during source initialization, including while it is opening. */
+    private readableFile;
+    /** Whether this source has been permanently closed. */
+    private closed;
+    /** Compression module preload shared by all selective reads. */
+    private compressionPromise;
+    /** Read abort controllers used to cancel active iterators when the source closes. */
+    private activeReadControllers;
+    /** Cumulative source telemetry counters. */
+    private telemetry;
+    /** Creates a Parquet source backed by strict URL ranges or Blob slices. */
+    constructor(data: string | Blob, options: ParquetSourceLoaderOptions, coreApi?: CoreAPI);
+    /** Returns the cached logical schema decoded from the Parquet footer. */
+    getSchema(options?: {
+        signal?: AbortSignal;
+    }): Promise<Schema>;
+    /** Returns normalized dataset, row-group, and column-chunk metadata. */
+    getMetadata(options?: ParquetMetadataRequestOptions): Promise<ParquetSourceMetadata>;
+    /** Returns a copy of cumulative transport, decode, conversion, and pruning telemetry. */
+    getTelemetry(): ParquetTelemetry;
+    /** Selectively fetches row groups and columns as ordered Arrow batches with source provenance. */
+    read(options?: ParquetSourceReadOptions): AsyncIterable<ParquetSourceBatch>;
+    /** Closes the underlying readable file and aborts active remote requests. */
+    close(): Promise<void>;
+    /** Returns cached initialization, resetting the cache when initialization fails. */
+    private getInitialization;
+    /** Preloads TypeScript decoder compression modules once and permits retry after failure. */
+    private getCompressionInitialization;
+    /** Snapshots source defaults and per-read overrides before asynchronous iteration begins. */
+    private getReadOptions;
+    /** Creates loader-utils worker options for one selective source read. */
+    private getWorkerOptions;
+    /** Fetches and decodes one selected row group using the cached footer and schema. */
+    private readRowGroup;
+    /** Fetches selected chunks and transfers their decompression and Arrow conversion to a worker. */
+    private readRowGroupOnWorker;
+    /** Records the rows and pages avoided by one conservative page-index plan. */
+    private recordPagePruningTelemetry;
+    /** Opens the readable file and decodes its footer and schema once. */
+    private initialize;
+    /** Opens a Blob file or probes a remote object with one bounded range request. */
+    private openFile;
+    /** Applies telemetry deltas and emits a callback with the resulting cumulative snapshot. */
+    private recordTelemetry;
+}
+//# sourceMappingURL=parquet-source-loader.d.ts.map

package/dist/parquet-source-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-source-loader.d.ts","sourceRoot":"","sources":["../src/parquet-source-loader.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,OAAO,EAA6B,MAAM,0BAA0B,CAAC;AAClF,OAAO,EAAW,UAAU,EAAY,MAAM,0BAA0B,CAAC;AACzE,OAAO,KAAK,EAAwB,MAAM,EAAC,MAAM,oBAAoB,CAAC;AA2BtE,OAAO,EAEL,KAAK,yBAAyB,EAC/B,yCAAsC;AAEvC,OAAO,KAAK,EAIV,6BAA6B,EAI7B,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAEjB,kCAA+B;AAChC,OAAO,EACL,2BAA2B,EAC3B,KAAK,yBAAyB,EAC/B,yCAAsC;AAyBvC,YAAY,EACV,YAAY,EACZ,oBAAoB,EACpB,sBAAsB,EACtB,0BAA0B,EAC1B,4BAA4B,EAC5B,6BAA6B,EAC7B,oBAAoB,EACpB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,mBAAmB,EACnB,oBAAoB,EACpB,gBAAgB,EAChB,wBAAwB,EACxB,qBAAqB,EACrB,0BAA0B,EAC1B,kBAAkB,EAClB,uBAAuB,EACvB,kBAAkB,EAClB,0BAA0B,EAC1B,qBAAqB,EACrB,wBAAwB,EACxB,gBAAgB,EAChB,qBAAqB,EACtB,kCAA+B;AAkDhC,wEAAwE;AACxE,eAAO,MAAM,6BAA6B;sCAGhC,MAAM,GAAG,IAAI,WACV,0BAA0B,YACzB,OAAO,KAChB,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAC8B,CAAC;AAEjD,6EAA6E;AAC7E,OAAO,EAAC,6BAA6B,IAAI,mBAAmB,EAAC,CAAC;AAE9D,iGAAiG;AACjG,qBAAa,aAAc,SAAQ,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,0BAA0B,CAAC;IACtF,oFAAoF;IACpF,QAAQ,CAAC,YAAY,EAAE,yBAAyB,CAA+B;IAC/E,sDAAsD;IACtD,OAAO,CAAC,qBAAqB,CAAqD;IAClF,kFAAkF;IAClF,OAAO,CAAC,YAAY,CAA6B;IACjD,uDAAuD;IACvD,OAAO,CAAC,MAAM,CAAS;IACvB,gEAAgE;IAChE,OAAO,CAAC,kBAAkB,CAA8B;IACxD,qFAAqF;IACrF,OAAO,CAAC,qBAAqB,CAA8B;IAC3D,4CAA4C;IAC5C,OAAO,CAAC,SAAS,CAA4B;IAE7C,2EAA2E;gBAC/D,IAAI,EAAE,MAAM,GAAG,IAAI,EAAE,OAAO,EAAE,0BAA0B,EAAE,OAAO,CAAC,EAAE,OAAO;IAQvF,yEAAyE;IACnE,SAAS,CAAC,OAAO,GAAE;QAAC,MAAM,CAAC,EAAE,WAAW,CAAA;KAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAKtE,wEAAwE;IAClE,WAAW,CAAC,OAAO,GAAE,6BAAkC,GAAG,OAAO,CAAC,qBAAqB,CAAC;IAW9F,yFAAyF;IACzF,YAAY,IAAI,gBAAgB;IAIhC,kGAAkG;IAC3F,IAAI,CAAC,OAAO,GAAE,wBAA6B,GAAG,aAAa,CAAC,kBAAkB,CAAC;IAgMtF,6EAA6E;IACvE,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAe5B,oFAAoF;IACpF,OAAO,CAAC,iBAAiB;IAazB,4FAA4F;IAC5F,OAAO,CAAC,4BAA4B;IAYpC,6FAA6F;IAC7F,OAAO,CAAC,cAAc;IAetB,yEAAyE;IACzE,OAAO,CAAC,gBAAgB;IAkBxB,qFAAqF;YACvE,YAAY;IA4G1B,kGAAkG;YACpF,oBAAoB;IAwElC,8EAA8E;IAC9E,OAAO,CAAC,0BAA0B;IAelC,sEAAsE;YACxD,UAAU;IA2BxB,kFAAkF;YACpE,QAAQ;IA4BtB,4FAA4F;IAC5F,OAAO,CAAC,eAAe;CAkBxB"}