@loaders.gl/parquet 4.4.5 → 5.0.0-alpha.2

package/dist/lib/parsers/parse-parquet-to-arrow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAS,MAAM,oBAAoB,CAAC;AAG5E,OAAO,KAAK,KAAK,WAAW,MAAM,cAAc,CAAC;AAMjD,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACtD,OAAO,CAAC,UAAU,CAAC,CAsBrB;AAED,wBAAuB,gCAAgC,CACrD,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACrD,aAAa,CAAC,eAAe,CAAC,CA0BhC"}
+{"version":3,"file":"parse-parquet-to-arrow.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAS,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,KAAK,WAAW,MAAM,kCAAkC,CAAC;AAUrE,KAAK,wBAAwB,GAAG,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAC,CAAC;AAE/E,wBAAsB,uBAAuB,CAC3C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,UAAU,CAAC,CAqBrB;AAED,wBAAuB,gCAAgC,CACrD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,aAAa,CAAC,eAAe,CAAC,CAgChC"}

package/dist/lib/parsers/parse-parquet-to-arrow.js

@@ -1,52 +1,65 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { convertArrowToSchema } from '@loaders.gl/schema-utils';
 import * as arrow from 'apache-arrow';
+import { normalizeArrowTableGeoMetadata, getParquetFileMetadataMap } from "../geo/geospatial-metadata.js";
 import { loadWasm } from "../utils/load-wasm.js";
 import { makeStreamIterator } from "../utils/make-stream-iterator.js";
 export async function parseParquetFileToArrow(file, options) {
     const wasmUrl = options?.wasmUrl;
+    const readerOptions = getReaderOptions(options);
     const wasm = await loadWasm(wasmUrl);
-    let parquetFile;
-    if (file.handle instanceof Blob) {
-        // TODO - let's assume fromFile() works on Blobs and not just on File...
-        parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+    const parquetFile = await createParquetFile(file, wasm);
+    try {
+        const wasmTable = await parquetFile.read(readerOptions);
+        const ipcStream = wasmTable.intoIPCStream();
+        const arrowTable = arrow.tableFromIPC(ipcStream);
+        return normalizeArrowTableGeoMetadata({
+            shape: 'arrow-table',
+            data: arrowTable
+        }, getParquetFileMetadataMap(parquetFile.metadata()));
     }
-    else {
-        parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+    finally {
+        parquetFile.free();
     }
-    const wasmTable = await parquetFile.read(options);
-    const ipcStream = wasmTable.intoIPCStream();
-    const arrowTable = arrow.tableFromIPC(ipcStream);
-    return {
-        shape: 'arrow-table',
-        schema: convertArrowToSchema(arrowTable.schema),
-        data: arrowTable
-    };
 }
 export async function* parseParquetFileToArrowInBatches(file, options) {
     const wasmUrl = options?.wasmUrl;
+    const readerOptions = getReaderOptions(options);
     const wasm = await loadWasm(wasmUrl);
-    let parquetFile;
-    if (file.handle instanceof Blob) {
-        // Works on Blobs: https://kylebarron.dev/parquet-wasm/classes/esm_parquet_wasm.ParquetFile.html#fromFile
-        parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+    const parquetFile = await createParquetFile(file, wasm);
+    try {
+        const stream = await parquetFile.stream(readerOptions);
+        const schemaMetadata = getParquetFileMetadataMap(parquetFile.metadata());
+        let schema;
+        for await (const wasmRecordBatch of makeStreamIterator(stream)) {
+            const normalizedArrowTable = normalizeArrowTableGeoMetadata({
+                shape: 'arrow-table',
+                data: arrow.tableFromIPC(wasmRecordBatch.intoIPCStream())
+            }, schemaMetadata);
+            schema ||= normalizedArrowTable.schema;
+            yield {
+                batchType: 'data',
+                shape: 'arrow-table',
+                schema,
+                data: normalizedArrowTable.data,
+                length: normalizedArrowTable.data.numRows
+            };
+        }
     }
-    else {
-        parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+    finally {
+        parquetFile.free();
     }
-    const stream = await parquetFile.stream(options);
-    let schema;
-    for await (const recordBatch of makeStreamIterator(stream)) {
-        schema ||= convertArrowToSchema(recordBatch.schema);
-        yield {
-            batchType: 'data',
-            shape: 'arrow-table',
-            schema,
-            data: new arrow.Table(recordBatch),
-            length: recordBatch.numRows
-        };
+}
+async function createParquetFile(file, wasm) {
+    if (file.handle instanceof Blob) {
+        // Works on Blobs: https://kylebarron.dev/parquet-wasm/classes/esm_parquet_wasm.ParquetFile.html#fromFile
+        return await wasm.ParquetFile.fromFile(file.handle);
     }
+    return await wasm.ParquetFile.fromUrl(file.url);
+}
+function getReaderOptions(options) {
+    const { wasmUrl, ...readerOptions } = options || {};
+    return readerOptions;
 }
 //# sourceMappingURL=parse-parquet-to-arrow.js.map

package/dist/lib/parsers/parse-parquet-to-arrow.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-arrow.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,OAAO,EAAC,oBAAoB,EAAC,MAAM,0BAA0B,CAAC;AAG9D,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAC,QAAQ,EAAC,8BAA2B;AAC5C,OAAO,EAAC,kBAAkB,EAAC,yCAAsC;AAEjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAkB,EAClB,OAAuD;IAEvD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAEjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAErC,IAAI,WAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,MAAM,YAAY,IAAI,EAAE,CAAC;QAChC,wEAAwE;QACxE,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAc,CAAC,CAAC;IACrE,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAClD,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC;IAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;IAEjD,OAAO;QACL,KAAK,EAAE,aAAa;QACpB,MAAM,EAAE,oBAAoB,CAAC,UAAU,CAAC,MAAM,CAAC;QAC/C,IAAI,EAAE,UAAU;KACjB,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gCAAgC,CACrD,IAAkB,EAClB,OAAsD;IAEtD,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAEjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAErC,IAAI,WAAoC,CAAC;IACzC,IAAI,IAAI,CAAC,MAAM,YAAY,IAAI,EAAE,CAAC;QAChC,yGAAyG;QACzG,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAc,CAAC,CAAC;IACrE,CAAC;SAAM,CAAC;QACN,WAAW,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACzD,CAAC;IAED,MAAM,MAAM,GAAsC,MAAM,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAEpF,IAAI,MAAc,CAAC;IACnB,IAAI,KAAK,EAAE,MAAM,WAAW,IAAI,kBAAkB,CAAC,MAAM,CAAC,EAAE,CAAC;QAC3D,MAAM,KAAK,oBAAoB,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM;YACJ,SAAS,EAAE,MAAM;YACjB,KAAK,EAAE,aAAa;YACpB,MAAM;YACN,IAAI,EAAE,IAAI,KAAK,CAAC,KAAK,CAAC,WAAW,CAAC;YAClC,MAAM,EAAE,WAAW,CAAC,OAAO;SAC5B,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"parse-parquet-to-arrow.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-arrow.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAOpC,OAAO,KAAK,KAAK,MAAM,cAAc,CAAC;AAEtC,OAAO,EACL,8BAA8B,EAC9B,yBAAyB,EAC1B,sCAAmC;AACpC,OAAO,EAAC,QAAQ,EAAC,8BAA2B;AAC5C,OAAO,EAAC,kBAAkB,EAAC,yCAAsC;AAIjE,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,IAAkB,EAClB,OAAkC;IAElC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IACjC,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAErC,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,WAAW,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,SAAS,CAAC,aAAa,EAAE,CAAC;QAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QACjD,OAAO,8BAA8B,CACnC;YACE,KAAK,EAAE,aAAa;YACpB,IAAI,EAAE,UAAU;SACjB,EACD,yBAAyB,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAClD,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,WAAW,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,gCAAgC,CACrD,IAAkB,EAClB,OAAkC;IAElC,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IACjC,MAAM,aAAa,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAEhD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,OAAO,CAAC,CAAC;IAErC,MAAM,WAAW,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACxD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACvD,MAAM,cAAc,GAAG,yBAAyB,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,CAAC;QAEzE,IAAI,MAA0B,CAAC;QAC/B,IAAI,KAAK,EAAE,MAAM,eAAe,IAAI,kBAAkB,CAA0B,MAAM,CAAC,EAAE,CAAC;YACxF,MAAM,oBAAoB,GAAG,8BAA8B,CACzD;gBACE,KAAK,EAAE,aAAa;gBACpB,IAAI,EAAE,KAAK,CAAC,YAAY,CAAC,eAAe,CAAC,aAAa,EAAE,CAAC;aAC1D,EACD,cAAc,CACf,CAAC;YACF,MAAM,KAAK,oBAAoB,CAAC,MAAM,CAAC;YACvC,MAAM;gBACJ,SAAS,EAAE,MAAM;gBACjB,KAAK,EAAE,aAAa;gBACpB,MAAM;gBACN,IAAI,EAAE,oBAAoB,CAAC,IAAI;gBAC/B,MAAM,EAAE,oBAAoB,CAAC,IAAI,CAAC,OAAO;aAC1C,CAAC;QACJ,CAAC;IACH,CAAC;YAAS,CAAC;QACT,WAAW,CAAC,IAAI,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,IAAkB,EAClB,IAAwB;IAExB,IAAI,IAAI,CAAC,MAAM,YAAY,IAAI,EAAE,CAAC;QAChC,yGAAyG;QACzG,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,MAAc,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAkC;IAC1D,MAAM,EAAC,OAAO,EAAE,GAAG,aAAa,EAAC,GAAG,OAAO,IAAI,EAAE,CAAC;IAClD,OAAO,aAAa,CAAC;AACvB,CAAC"}

package/dist/lib/parsers/parse-parquet-to-columns.d.ts

@@ -1,12 +1,12 @@
 import type { ColumnarTable, ColumnarTableBatch } from '@loaders.gl/schema';
 import type { ReadableFile } from '@loaders.gl/loader-utils';
-import type { ParquetJSONLoaderOptions } from "../../parquet-json-loader.js";
+import type { ParquetLoaderOptions } from "../../parquet-loader.js";
 /**
  * @deprecated - Use parseParquetToArrow
  */
-export declare function parseParquetFileInColumns(file: ReadableFile, options?: ParquetJSONLoaderOptions): Promise<ColumnarTable>;
+export declare function parseParquetFileInColumns(file: ReadableFile, options?: ParquetLoaderOptions): Promise<ColumnarTable>;
 /**
  * @deprecated
  */
-export declare function parseParquetFileInColumnarBatches(file: ReadableFile, options?: ParquetJSONLoaderOptions): AsyncIterable<ColumnarTableBatch>;
+export declare function parseParquetFileInColumnarBatches(file: ReadableFile, options?: ParquetLoaderOptions): AsyncIterable<ColumnarTableBatch>;
 //# sourceMappingURL=parse-parquet-to-columns.d.ts.map

package/dist/lib/parsers/parse-parquet-to-columns.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,wBAAwB,EAAC,qCAAkC;AASxE;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,aAAa,CAAC,CAYxB;AAED;;GAEG;AACH,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,aAAa,CAAC,kBAAkB,CAAC,CAgBnC"}
+{"version":3,"file":"parse-parquet-to-columns.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,aAAa,EAAE,kBAAkB,EAAS,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAQ/D;;GAEG;AACH,wBAAsB,yBAAyB,CAC7C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,aAAa,CAAC,CAWxB;AAED;;GAEG;AACH,wBAAuB,iCAAiC,CACtD,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,kBAAkB,CAAC,CAiBnC"}

package/dist/lib/parsers/parse-parquet-to-columns.js

@@ -4,13 +4,11 @@
 import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { materializeColumns } from "../../parquetjs/schema/shred.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
-import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
 import { preloadCompressions } from "../../parquetjs/compression.js";
 /**
  * @deprecated - Use parseParquetToArrow
  */
 export async function parseParquetFileInColumns(file, options) {
-    installBufferPolyfill();
     await preloadCompressions(options);
     for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
         return {
@@ -25,14 +23,13 @@ export async function parseParquetFileInColumns(file, options) {
  * @deprecated
  */
 export async function* parseParquetFileInColumnarBatches(file, options) {
-    installBufferPolyfill();
     await preloadCompressions(options);
     const reader = new ParquetReader(file);
     // Extract schema and geo metadata
     const schema = await getSchemaFromParquetReader(reader);
     const parquetSchema = await reader.getSchema();
     // Iterate over row batches
-    const rowGroups = reader.rowGroupIterator(options?.parquet);
+    const rowGroups = reader.rowGroupIterator(options?.parquet?.columns?.length ? { columnList: options.parquet.columns } : undefined);
     for await (const rowGroup of rowGroups) {
         yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);
     }

package/dist/lib/parsers/parse-parquet-to-columns.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-columns.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,OAAO,EAAC,aAAa,EAAC,iDAA8C;AAGpE,OAAO,EAAC,kBAAkB,EAAC,wCAAqC;AAChE,OAAO,EAAC,0BAA0B,EAAC,gCAA6B;AAChE,OAAO,EAAC,qBAAqB,EAAC,wCAAqC;AACnE,OAAO,EAAC,mBAAmB,EAAC,uCAAoC;AAEhE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAkB,EAClB,OAAkC;IAElC,qBAAqB,EAAE,CAAC;IACxB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,iCAAiC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iCAAiC,CACtD,IAAkB,EAClB,OAAkC;IAElC,qBAAqB,EAAE,CAAC;IACxB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;IAEvC,kCAAkC;IAClC,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAExD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE/C,2BAA2B;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC5D,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACvC,MAAM,2BAA2B,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAClC,QAAyB,EACzB,aAA4B,EAC5B,MAAc;IAEd,kEAAkE;IAClE,MAAM,IAAI,GAAG,kBAAkB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IACzD,OAAO;QACL,KAAK,EAAE,gBAAgB;QACvB,SAAS,EAAE,MAAM;QACjB,MAAM;QACN,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,QAAQ;KAC1B,CAAC;AACJ,CAAC"}
+{"version":3,"file":"parse-parquet-to-columns.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAKpC,OAAO,EAAC,aAAa,EAAC,iDAA8C;AAGpE,OAAO,EAAC,kBAAkB,EAAC,wCAAqC;AAChE,OAAO,EAAC,0BAA0B,EAAC,gCAA6B;AAChE,OAAO,EAAC,mBAAmB,EAAC,uCAAoC;AAEhE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,yBAAyB,CAC7C,IAAkB,EAClB,OAA8B;IAE9B,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,iCAAiC,CAAC,IAAI,EAAE,OAAO,CAAC,EAAE,CAAC;QAC3E,OAAO;YACL,KAAK,EAAE,gBAAgB;YACvB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,IAAI,EAAE,KAAK,CAAC,IAAI;SACjB,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,aAAa,CAAC,CAAC;AACjC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,iCAAiC,CACtD,IAAkB,EAClB,OAA8B;IAE9B,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC;IAEvC,kCAAkC;IAClC,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAExD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAE/C,2BAA2B;IAC3B,MAAM,SAAS,GAAG,MAAM,CAAC,gBAAgB,CACvC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,EAAC,UAAU,EAAE,OAAO,CAAC,OAAO,CAAC,OAAO,EAAC,CAAC,CAAC,CAAC,SAAS,CACtF,CAAC;IACF,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACvC,MAAM,2BAA2B,CAAC,QAAQ,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IACrE,CAAC;AACH,CAAC;AAED,SAAS,2BAA2B,CAClC,QAAyB,EACzB,aAA4B,EAC5B,MAAc;IAEd,kEAAkE;IAClE,MAAM,IAAI,GAAG,kBAAkB,CAAC,aAAa,EAAE,QAAQ,CAAC,CAAC;IACzD,OAAO;QACL,KAAK,EAAE,gBAAgB;QACvB,SAAS,EAAE,MAAM;QACjB,MAAM;QACN,IAAI;QACJ,MAAM,EAAE,QAAQ,CAAC,QAAQ;KAC1B,CAAC;AACJ,CAAC"}

package/dist/lib/parsers/parse-parquet-to-json.d.ts

@@ -1,17 +1,17 @@
 import type { ReadableFile } from '@loaders.gl/loader-utils';
 import type { ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
-import type { ParquetJSONLoaderOptions } from "../../parquet-json-loader.js";
+import type { ParquetJSLoaderOptions } from "../../parquet-loader-options.js";
 /**
  *  * Parse a parquet file using parquetjs
  * @param file
  * @param options
  * @returns
  */
-export declare function parseParquetFile(file: ReadableFile, options?: ParquetJSONLoaderOptions): Promise<ObjectRowTable>;
+export declare function parseParquetFile(file: ReadableFile, options?: ParquetJSLoaderOptions): Promise<ObjectRowTable>;
 /**
  * Parse a parquet file in batches using parquetjs
  * @param file
  * @param options
  */
-export declare function parseParquetFileInBatches(file: ReadableFile, options?: ParquetJSONLoaderOptions): AsyncIterable<ObjectRowTableBatch>;
+export declare function parseParquetFileInBatches(file: ReadableFile, options?: ParquetJSLoaderOptions): AsyncIterable<ObjectRowTableBatch>;
 //# sourceMappingURL=parse-parquet-to-json.d.ts.map

package/dist/lib/parsers/parse-parquet-to-json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-json.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,wBAAwB,EAAC,qCAAkC;AAOxE;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,OAAO,CAAC,cAAc,CAAC,CAqCzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,wBAAwB,GACjC,aAAa,CAAC,mBAAmB,CAAC,CAwBpC"}
+{"version":3,"file":"parse-parquet-to-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-json.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAS,MAAM,oBAAoB,CAAC;AAEpF,OAAO,KAAK,EAAC,sBAAsB,EAAC,wCAAqC;AAMzE;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,OAAO,CAAC,cAAc,CAAC,CA2CzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,sBAAsB,GAC/B,aAAa,CAAC,mBAAmB,CAAC,CAsDpC"}

package/dist/lib/parsers/parse-parquet-to-json.js

@@ -4,7 +4,6 @@
 import { default as log } from '@probe.gl/log';
 import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
-import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
 import { preloadCompressions } from "../../parquetjs/compression.js";
 /**
  *  * Parse a parquet file using parquetjs
@@ -13,32 +12,37 @@ import { preloadCompressions } from "../../parquetjs/compression.js";
  * @returns
  */
 export async function parseParquetFile(file, options) {
-    installBufferPolyfill();
     await preloadCompressions(options);
     const reader = new ParquetReader(file, {
         preserveBinary: options?.parquet?.preserveBinary
     });
     const schema = await getSchemaFromParquetReader(reader);
+    const projectedSchema = projectSchema(schema, options?.parquet?.columns);
     const rows = [];
-    const rowBatches = reader.rowBatchIterator(options?.parquet);
+    const rowBatches = reader.rowBatchIterator(getParquetIterationProps(options));
+    const rowOffset = options?.parquet?.offset || 0;
+    const rowLimit = options?.parquet?.limit;
+    let rowsSkipped = 0;
     for await (const rowBatch of rowBatches) {
-        let limitHasReached = false;
-        // we have only one input batch so return
         for (const row of rowBatch) {
-            if (options?.limit && rows.length >= options?.core?.limit) {
-                limitHasReached = true;
-                break;
+            if (rowsSkipped < rowOffset) {
+                rowsSkipped++;
+                continue;
+            }
+            if (rowLimit !== undefined && rows.length >= rowLimit) {
+                log.warn(`Rows number limit has been reached. Only first ${rowLimit} are loaded`)();
+                return {
+                    shape: 'object-row-table',
+                    schema: projectedSchema,
+                    data: rows
+                };
             }
             rows.push(row);
         }
-        if (limitHasReached) {
-            log.warn(`Rows number limit has been reached. Only first ${options?.core?.limit} are loaded`)();
-            break;
-        }
     }
     const objectRowTable = {
         shape: 'object-row-table',
-        schema,
+        schema: projectedSchema,
         data: rows
     };
     return objectRowTable;
@@ -49,25 +53,72 @@ export async function parseParquetFile(file, options) {
  * @param options
  */
 export async function* parseParquetFileInBatches(file, options) {
-    installBufferPolyfill();
     await preloadCompressions(options);
     const reader = new ParquetReader(file, {
         preserveBinary: options?.parquet?.preserveBinary
     });
     const schema = await getSchemaFromParquetReader(reader);
-    const rowBatches = reader.rowBatchIterator(options?.parquet);
+    const projectedSchema = projectSchema(schema, options?.parquet?.columns);
+    const rowBatches = reader.rowBatchIterator(getParquetIterationProps(options));
+    const rowOffset = options?.parquet?.offset || 0;
+    const rowLimit = options?.parquet?.limit;
+    const outputBatchSize = options?.parquet?.batchSize;
+    let rowsSkipped = 0;
+    let rowsYielded = 0;
+    let reachedLimit = false;
     for await (const rows of rowBatches) {
-        const objectRowTable = {
-            shape: 'object-row-table',
-            schema,
-            data: rows
-        };
-        yield {
-            batchType: 'data',
-            schema,
-            ...objectRowTable,
-            length: rows.length
-        };
+        const nextRows = [];
+        for (const row of rows) {
+            if (rowsSkipped < rowOffset) {
+                rowsSkipped++;
+                continue;
+            }
+            if (rowLimit !== undefined && rowsYielded >= rowLimit) {
+                reachedLimit = true;
+                break;
+            }
+            nextRows.push(row);
+            rowsYielded++;
+        }
+        if (nextRows.length === 0) {
+            continue;
+        }
+        if (!outputBatchSize || outputBatchSize <= 0) {
+            yield makeObjectRowBatch(projectedSchema, nextRows);
+            continue;
+        }
+        for (let rowIndex = 0; rowIndex < nextRows.length; rowIndex += outputBatchSize) {
+            yield makeObjectRowBatch(projectedSchema, nextRows.slice(rowIndex, rowIndex + outputBatchSize));
+        }
+        if (reachedLimit) {
+            return;
+        }
     }
 }
+function getParquetIterationProps(options) {
+    const columnList = options?.parquet?.columns;
+    return columnList?.length ? { columnList } : undefined;
+}
+function makeObjectRowBatch(schema, rows) {
+    const objectRowTable = {
+        shape: 'object-row-table',
+        schema,
+        data: rows
+    };
+    return {
+        batchType: 'data',
+        schema,
+        ...objectRowTable,
+        length: rows.length
+    };
+}
+function projectSchema(schema, columns) {
+    if (!columns?.length) {
+        return schema;
+    }
+    return {
+        ...schema,
+        fields: schema.fields.filter(field => columns.includes(field.name))
+    };
+}
 //# sourceMappingURL=parse-parquet-to-json.js.map

package/dist/lib/parsers/parse-parquet-to-json.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet-to-json.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-json.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,OAAO,IAAI,GAAG,EAAC,MAAM,eAAe,CAAC;AAM7C,OAAO,EAAC,aAAa,EAAC,iDAA8C;AACpE,OAAO,EAAC,0BAA0B,EAAC,gCAA6B;AAChE,OAAO,EAAC,qBAAqB,EAAC,wCAAqC;AACnE,OAAO,EAAC,mBAAmB,EAAC,uCAAoC;AAEhE;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAkB,EAClB,OAAkC;IAElC,qBAAqB,EAAE,CAAC;IACxB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc;KACjD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IAExD,MAAM,IAAI,GAAiB,EAAE,CAAC;IAE9B,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7D,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QACxC,IAAI,eAAe,GAAG,KAAK,CAAC;QAC5B,yCAAyC;QACzC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,OAAO,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,EAAE,IAAI,EAAE,KAAM,EAAE,CAAC;gBAC3D,eAAe,GAAG,IAAI,CAAC;gBACvB,MAAM;YACR,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,IAAI,eAAe,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CACN,kDAAkD,OAAO,EAAE,IAAI,EAAE,KAAK,aAAa,CACpF,EAAE,CAAC;YACJ,MAAM;QACR,CAAC;IACH,CAAC;IACD,MAAM,cAAc,GAAmB;QACrC,KAAK,EAAE,kBAAkB;QACzB,MAAM;QACN,IAAI,EAAE,IAAI;KACX,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,yBAAyB,CAC9C,IAAkB,EAClB,OAAkC;IAElC,qBAAqB,EAAE,CAAC;IACxB,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc;KACjD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC7D,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,cAAc,GAAmB;YACrC,KAAK,EAAE,kBAAkB;YACzB,MAAM;YACN,IAAI,EAAE,IAAI;SACX,CAAC;QAEF,MAAM;YACJ,SAAS,EAAE,MAAM;YACjB,MAAM;YACN,GAAG,cAAc;YACjB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC;IACJ,CAAC;AACH,CAAC"}
+{"version":3,"file":"parse-parquet-to-json.js","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-json.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,+BAA+B;AAC/B,oCAAoC;AAEpC,OAAO,EAAC,OAAO,IAAI,GAAG,EAAC,MAAM,eAAe,CAAC;AAM7C,OAAO,EAAC,aAAa,EAAC,iDAA8C;AACpE,OAAO,EAAC,0BAA0B,EAAC,gCAA6B;AAChE,OAAO,EAAC,mBAAmB,EAAC,uCAAoC;AAEhE;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAkB,EAClB,OAAgC;IAEhC,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc;KACjD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAEzE,MAAM,IAAI,GAAiB,EAAE,CAAC;IAE9B,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC;IACzC,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,KAAK,EAAE,MAAM,QAAQ,IAAI,UAAU,EAAE,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;gBAC5B,WAAW,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;gBACtD,GAAG,CAAC,IAAI,CAAC,kDAAkD,QAAQ,aAAa,CAAC,EAAE,CAAC;gBACpF,OAAO;oBACL,KAAK,EAAE,kBAAkB;oBACzB,MAAM,EAAE,eAAe;oBACvB,IAAI,EAAE,IAAI;iBACX,CAAC;YACJ,CAAC;YAED,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,cAAc,GAAmB;QACrC,KAAK,EAAE,kBAAkB;QACzB,MAAM,EAAE,eAAe;QACvB,IAAI,EAAE,IAAI;KACX,CAAC;IAEF,OAAO,cAAc,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,yBAAyB,CAC9C,IAAkB,EAClB,OAAgC;IAEhC,MAAM,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAEnC,MAAM,MAAM,GAAG,IAAI,aAAa,CAAC,IAAI,EAAE;QACrC,cAAc,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc;KACjD,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,MAAM,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACxD,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IACzE,MAAM,UAAU,GAAG,MAAM,CAAC,gBAAgB,CAAC,wBAAwB,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9E,MAAM,SAAS,GAAG,OAAO,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,OAAO,EAAE,OAAO,EAAE,KAAK,CAAC;IACzC,MAAM,eAAe,GAAG,OAAO,EAAE,OAAO,EAAE,SAAS,CAAC;IACpD,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,YAAY,GAAG,KAAK,CAAC;IAEzB,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QACpC,MAAM,QAAQ,GAAiB,EAAE,CAAC;QAClC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,IAAI,WAAW,GAAG,SAAS,EAAE,CAAC;gBAC5B,WAAW,EAAE,CAAC;gBACd,SAAS;YACX,CAAC;YAED,IAAI,QAAQ,KAAK,SAAS,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC;gBACtD,YAAY,GAAG,IAAI,CAAC;gBACpB,MAAM;YACR,CAAC;YAED,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,WAAW,EAAE,CAAC;QAChB,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,SAAS;QACX,CAAC;QAED,IAAI,CAAC,eAAe,IAAI,eAAe,IAAI,CAAC,EAAE,CAAC;YAC7C,MAAM,kBAAkB,CAAC,eAAe,EAAE,QAAQ,CAAC,CAAC;YACpD,SAAS;QACX,CAAC;QAED,KAAK,IAAI,QAAQ,GAAG,CAAC,EAAE,QAAQ,GAAG,QAAQ,CAAC,MAAM,EAAE,QAAQ,IAAI,eAAe,EAAE,CAAC;YAC/E,MAAM,kBAAkB,CACtB,eAAe,EACf,QAAQ,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,GAAG,eAAe,CAAC,CACrD,CAAC;QACJ,CAAC;QAED,IAAI,YAAY,EAAE,CAAC;YACjB,OAAO;QACT,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAC/B,OAAgC;IAEhC,MAAM,UAAU,GAAG,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC;IAC7C,OAAO,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,EAAC,UAAU,EAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvD,CAAC;AAED,SAAS,kBAAkB,CAAC,MAAc,EAAE,IAAkB;IAC5D,MAAM,cAAc,GAAmB;QACrC,KAAK,EAAE,kBAAkB;QACzB,MAAM;QACN,IAAI,EAAE,IAAI;KACX,CAAC;IAEF,OAAO;QACL,SAAS,EAAE,MAAM;QACjB,MAAM;QACN,GAAG,cAAc;QACjB,MAAM,EAAE,IAAI,CAAC,MAAM;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,MAAc,EAAE,OAAkB;IACvD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,OAAO;QACL,GAAG,MAAM;QACT,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;KACpE,CAAC;AACJ,CAAC"}

package/dist/lib/sources/parquet-range-file.d.ts

@@ -0,0 +1,80 @@
+import type { ReadableFile, Stat } from '@loaders.gl/loader-utils';
+import type { ParquetObjectVersion, ParquetRangeRequestOptions } from "../../parquet-source-types.js";
+type FetchLike = (url: string, options?: RequestInit) => Promise<Response>;
+type ParquetRangeFileOptions = {
+    /** Fetch implementation used for every HTTP range. */
+    fetch: FetchLike;
+    /** Headers forwarded to every HTTP range. */
+    headers?: HeadersInit;
+    /** Range coalescing and diagnostics configuration. */
+    rangeRequests?: ParquetRangeRequestOptions;
+    /** Receives source-local transport telemetry deltas. */
+    onTelemetry?: (event: ParquetRangeTelemetryEvent) => void;
+};
+type ParquetRangeTelemetryEvent = {
+    /** Transport operation that produced this event. */
+    type: 'range-request' | 'cache-hit';
+    /** Number of transport requests contributed by this event. */
+    rangeRequestCount?: number;
+    /** Number of transport bytes requested by this event. */
+    requestedBytes?: number;
+    /** Number of response bytes downloaded by this event. */
+    downloadedBytes?: number;
+    /** Number of cache hits contributed by this event. */
+    cacheHits?: number;
+    /** Network duration contributed by this event. */
+    networkDurationMs?: number;
+    /** Number of failed requests contributed by this event. */
+    failedRangeRequestCount?: number;
+    /** Number of aborted requests contributed by this event. */
+    abortedRangeRequestCount?: number;
+    /** Error associated with a failed range request. */
+    error?: unknown;
+};
+/** Strict HTTP range-backed file with Parquet object-version validation. */
+export declare class ParquetRangeFile implements ReadableFile {
+    /** URL used as the file handle. */
+    readonly handle: string;
+    /** URL of the remote Parquet object. */
+    readonly url: string;
+    /** Fetch implementation inherited from source loader options. */
+    private readonly fetch;
+    /** Headers inherited from source loader options. */
+    private readonly headers?;
+    /** Source telemetry callback invoked after transport and cache operations. */
+    private readonly onTelemetry?;
+    /** Scheduler used to coalesce later row-group reads. */
+    private readonly scheduler;
+    /** Aborts all active requests when the file is closed. */
+    private readonly closeController;
+    /** Exact ranges already loaded during initialization. */
+    private readonly cache;
+    /** Remote object byte length discovered from `Content-Range`. */
+    private fileByteLength;
+    /** Object validators captured by the opening range request. */
+    private version;
+    /** Whether `close()` has been called. */
+    private closed;
+    /** Creates an unopened remote Parquet file. Use `open()` before reading it. */
+    constructor(url: string, options: ParquetRangeFileOptions);
+    /** Total remote object byte length after `open()`. */
+    get size(): number;
+    /** Total remote object byte length as a bigint after `open()`. */
+    get bigsize(): bigint;
+    /** Immutable copy of the HTTP validators captured when the object was opened. */
+    get objectVersion(): ParquetObjectVersion;
+    /** Opens the object with a four-byte range probe and caches the Parquet header. */
+    open(signal?: AbortSignal): Promise<this>;
+    /** Reads an exact byte range without downloading the complete object. */
+    read(start?: number | bigint, length?: number, signal?: AbortSignal): Promise<ArrayBuffer>;
+    /** Returns file length information without another HTTP request. */
+    stat(): Promise<Stat>;
+    /** Aborts active requests and prevents future reads. */
+    close(): Promise<void>;
+    /** Fetches and validates one exact HTTP byte range. */
+    private fetchExactRange;
+    /** Throws when the file has already been closed. */
+    private assertOpen;
+}
+export {};
+//# sourceMappingURL=parquet-range-file.d.ts.map

package/dist/lib/sources/parquet-range-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet-range-file.d.ts","sourceRoot":"","sources":["../../../src/lib/sources/parquet-range-file.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAE,IAAI,EAAC,MAAM,0BAA0B,CAAC;AAGjE,OAAO,KAAK,EAAC,oBAAoB,EAAE,0BAA0B,EAAC,sCAAmC;AAEjG,KAAK,SAAS,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,WAAW,KAAK,OAAO,CAAC,QAAQ,CAAC,CAAC;AAE3E,KAAK,uBAAuB,GAAG;IAC7B,sDAAsD;IACtD,KAAK,EAAE,SAAS,CAAC;IACjB,6CAA6C;IAC7C,OAAO,CAAC,EAAE,WAAW,CAAC;IACtB,sDAAsD;IACtD,aAAa,CAAC,EAAE,0BAA0B,CAAC;IAC3C,wDAAwD;IACxD,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;CAC3D,CAAC;AAEF,KAAK,0BAA0B,GAAG;IAChC,oDAAoD;IACpD,IAAI,EAAE,eAAe,GAAG,WAAW,CAAC;IACpC,8DAA8D;IAC9D,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,yDAAyD;IACzD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,yDAAyD;IACzD,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kDAAkD;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,2DAA2D;IAC3D,uBAAuB,CAAC,EAAE,MAAM,CAAC;IACjC,4DAA4D;IAC5D,wBAAwB,CAAC,EAAE,MAAM,CAAC;IAClC,oDAAoD;IACpD,KAAK,CAAC,EAAE,OAAO,CAAC;CACjB,CAAC;AAQF,4EAA4E;AAC5E,qBAAa,gBAAiB,YAAW,YAAY;IACnD,mCAAmC;IACnC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB,iEAAiE;IACjE,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAY;IAClC,oDAAoD;IACpD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAc;IACvC,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC,CAA8C;IAC3E,wDAAwD;IACxD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAwB;IAClD,0DAA0D;IAC1D,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAyB;IACzD,yDAAyD;IACzD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAkC;IACxD,iEAAiE;IACjE,OAAO,CAAC,cAAc,CAAK;IAC3B,+DAA+D;IAC/D,OAAO,CAAC,OAAO,CAA4B;IAC3C,yCAAyC;IACzC,OAAO,CAAC,MAAM,CAAS;IAEvB,+EAA+E;gBACnE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,uBAAuB;IAkBzD,sDAAsD;IACtD,IAAI,IAAI,IAAI,MAAM,CAEjB;IAED,kEAAkE;IAClE,IAAI,OAAO,IAAI,MAAM,CAEpB;IAED,iFAAiF;IACjF,IAAI,aAAa,IAAI,oBAAoB,CAExC;IAED,mFAAmF;IAC7E,IAAI,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC;IAS/C,yEAAyE;IACnE,IAAI,CACR,KAAK,GAAE,MAAM,GAAG,MAAU,EAC1B,MAAM,GAAE,MAAkC,EAC1C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC,WAAW,CAAC;IAwCvB,oEAAoE;IAC9D,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAK3B,wDAAwD;IAClD,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAQ5B,uDAAuD;YACzC,eAAe;IAqE7B,oDAAoD;IACpD,OAAO,CAAC,UAAU;CAKnB"}