@loaders.gl/parquet 4.2.1 → 4.3.0-alpha.2

package/dist/lib/constants.d.ts

@@ -1,5 +1,5 @@
 export declare const VERSION: any;
-export declare const PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm";
+export declare const PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
 /**
  * Parquet File Magic String
  */

package/dist/lib/constants.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/lib/constants.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,OAAO,KAA8D,CAAC;AACnF,eAAO,MAAM,gBAAgB,mEAAmE,CAAC;AAEjG;;GAEG;AACH,eAAO,MAAM,aAAa,SAAS,CAAC;AACpC,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;GAEG;AACH,eAAO,MAAM,kBAAkB,UAAU,CAAC;AAC1C,eAAO,MAAM,sBAAsB,QAAQ,CAAC"}
+{"version":3,"file":"constants.d.ts","sourceRoot":"","sources":["../../src/lib/constants.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,OAAO,KAA8D,CAAC;AAEnF,eAAO,MAAM,gBAAgB,kEAAkE,CAAC;AAEhG;;GAEG;AACH,eAAO,MAAM,aAAa,SAAS,CAAC;AACpC,eAAO,MAAM,uBAAuB,SAAS,CAAC;AAE9C;;GAEG;AACH,eAAO,MAAM,eAAe,IAAI,CAAC;AAEjC;;GAEG;AACH,eAAO,MAAM,kBAAkB,UAAU,CAAC;AAC1C,eAAO,MAAM,sBAAsB,QAAQ,CAAC"}

package/dist/lib/constants.js

@@ -3,8 +3,8 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-export const VERSION = typeof "4.2.0" !== 'undefined' ? "4.2.0" : 'latest';
-export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
+export const VERSION = typeof "4.3.0-alpha.1" !== 'undefined' ? "4.3.0-alpha.1" : 'latest';
+export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm';
 /**
  * Parquet File Magic String
  */

package/dist/lib/encoders/encode-parquet-wasm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/encoders/encode-parquet-wasm.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,OAAO,KAAK,EAAC,oBAAoB,EAAC,qCAAkC;AAEpE;;GAEG;AACH,wBAAsB,MAAM,CAC1B,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAsBtB"}

package/dist/lib/{wasm → encoders}/encode-parquet-wasm.js

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 import * as arrow from 'apache-arrow';
-import { loadWasm } from "./load-wasm.js";
+import { loadWasm } from "../utils/load-wasm.js";
 /**
  * Encode Arrow arrow.Table to Parquet buffer
  */

package/dist/lib/parsers/parse-parquet-wasm.d.ts

@@ -0,0 +1,10 @@
+import type { ReadableFile } from '@loaders.gl/loader-utils';
+import type { ArrowTable, ArrowTableBatch } from '@loaders.gl/schema';
+import type * as parquetWasm from 'parquet-wasm';
+export declare function parseParquetFileWasm(file: ReadableFile, options?: parquetWasm.ReaderOptions & {
+    wasmUrl: string;
+}): Promise<ArrowTable>;
+export declare function parseParquetFileInBatchesWasm(file: ReadableFile, options: parquetWasm.ReaderOptions & {
+    wasmUrl: string;
+}): AsyncIterable<ArrowTableBatch>;
+//# sourceMappingURL=parse-parquet-wasm.d.ts.map

package/dist/lib/parsers/parse-parquet-wasm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-wasm.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAS,MAAM,oBAAoB,CAAC;AAG5E,OAAO,KAAK,KAAK,WAAW,MAAM,cAAc,CAAC;AAMjD,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACtD,OAAO,CAAC,UAAU,CAAC,CAsBrB;AAED,wBAAuB,6BAA6B,CAClD,IAAI,EAAE,YAAY,EAClB,OAAO,EAAE,WAAW,CAAC,aAAa,GAAG;IAAC,OAAO,EAAE,MAAM,CAAA;CAAC,GACrD,aAAa,CAAC,eAAe,CAAC,CA0BhC"}

package/dist/lib/parsers/parse-parquet-wasm.js

@@ -0,0 +1,51 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { serializeArrowSchema } from '@loaders.gl/arrow';
+import * as arrow from 'apache-arrow';
+import { loadWasm } from "../utils/load-wasm.js";
+import { makeStreamIterator } from "../utils/make-stream-iterator.js";
+export async function parseParquetFileWasm(file, options) {
+    const wasmUrl = options?.wasmUrl;
+    const wasm = await loadWasm(wasmUrl);
+    let parquetFile;
+    if (file.handle instanceof Blob) {
+        // TODO - let's assume fromFile() works on Blobs and not just on File...
+        parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+    }
+    else {
+        parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+    }
+    const wasmTable = await parquetFile.read(options);
+    const ipcStream = wasmTable.intoIPCStream();
+    const arrowTable = arrow.tableFromIPC(ipcStream);
+    return {
+        shape: 'arrow-table',
+        schema: serializeArrowSchema(arrowTable.schema),
+        data: arrowTable
+    };
+}
+export async function* parseParquetFileInBatchesWasm(file, options) {
+    const wasmUrl = options?.wasmUrl;
+    const wasm = await loadWasm(wasmUrl);
+    let parquetFile;
+    if (file.handle instanceof Blob) {
+        // Works on Blobs: https://kylebarron.dev/parquet-wasm/classes/esm_parquet_wasm.ParquetFile.html#fromFile
+        parquetFile = await wasm.ParquetFile.fromFile(file.handle);
+    }
+    else {
+        parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+    }
+    const stream = await parquetFile.stream(options);
+    let schema;
+    for await (const table of makeStreamIterator(stream)) {
+        schema ||= serializeArrowSchema(table.schema);
+        yield {
+            batchType: 'data',
+            shape: 'arrow-table',
+            schema,
+            data: table.batches[0],
+            length: table.numRows
+        };
+    }
+}

package/dist/lib/parsers/parse-parquet.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CA2BzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}
+{"version":3,"file":"parse-parquet.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EAAC,cAAc,EAAE,mBAAmB,EAAC,MAAM,oBAAoB,CAAC;AAE5E,OAAO,KAAK,EAAC,oBAAoB,EAAC,gCAA6B;AAO/D;;;;;GAKG;AACH,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,CAAC,CAoCzB;AAED;;;;GAIG;AACH,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,CAAC,CA0BpC"}

package/dist/lib/parsers/parse-parquet.js

@@ -1,6 +1,7 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
+import { default as log } from '@probe.gl/log';
 import { ParquetReader } from "../../parquetjs/parser/parquet-reader.js";
 import { getSchemaFromParquetReader } from "./get-parquet-schema.js";
 import { installBufferPolyfill } from "../../polyfills/buffer/index.js";
@@ -21,10 +22,19 @@ export async function parseParquetFile(file, options) {
     const rows = [];
     const rowBatches = reader.rowBatchIterator(options?.parquet);
     for await (const rowBatch of rowBatches) {
+        let limitHasReached = false;
         // we have only one input batch so return
         for (const row of rowBatch) {
+            if (options?.limit && rows.length >= options?.limit) {
+                limitHasReached = true;
+                break;
+            }
             rows.push(row);
         }
+        if (limitHasReached) {
+            log.warn(`Rows number limit has been reached. Only first ${options?.limit} are loaded`)();
+            break;
+        }
     }
     const objectRowTable = {
         shape: 'object-row-table',

package/dist/lib/utils/load-wasm.d.ts

@@ -0,0 +1,3 @@
+import * as parquetWasm from 'parquet-wasm';
+export declare function loadWasm(wasmUrl?: string): Promise<typeof parquetWasm>;
+//# sourceMappingURL=load-wasm.d.ts.map

package/dist/lib/utils/load-wasm.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"load-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/load-wasm.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,WAAW,MAAM,cAAc,CAAC;AAK5C,wBAAsB,QAAQ,CAAC,OAAO,GAAE,MAAyB,+BAUhE"}

package/dist/lib/utils/make-stream-iterator.d.ts

@@ -0,0 +1,11 @@
+/// <reference types="node" />
+import type { Readable } from 'stream';
+export type StreamIteratorOptions = {
+    _streamReadAhead?: boolean;
+};
+/**
+ * Returns an async iterable that reads from a stream (works in both Node.js and browsers)
+ * @param stream stream to iterator over
+ */
+export declare function makeStreamIterator<T>(stream: ReadableStream<T> | Readable, options?: StreamIteratorOptions): AsyncIterable<T>;
+//# sourceMappingURL=make-stream-iterator.d.ts.map

package/dist/lib/utils/make-stream-iterator.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"make-stream-iterator.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/make-stream-iterator.ts"],"names":[],"mappings":";AAIA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,QAAQ,CAAC;AAGrC,MAAM,MAAM,qBAAqB,GAAG;IAClC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B,CAAC;AAEF;;;GAGG;AACH,wBAAgB,kBAAkB,CAAC,CAAC,EAClC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,QAAQ,EACpC,OAAO,CAAC,EAAE,qBAAqB,GAC9B,aAAa,CAAC,CAAC,CAAC,CAIlB"}

package/dist/lib/utils/make-stream-iterator.js

@@ -0,0 +1,67 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import { isBrowser } from '@loaders.gl/loader-utils';
+/**
+ * Returns an async iterable that reads from a stream (works in both Node.js and browsers)
+ * @param stream stream to iterator over
+ */
+export function makeStreamIterator(stream, options) {
+    return isBrowser
+        ? makeBrowserStreamIterator(stream, options)
+        : makeNodeStreamIterator(stream, options);
+}
+/**
+ * Returns an async iterable that reads from a DOM (browser) stream
+ * @param stream stream to iterate from
+ * @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate
+ */
+async function* makeBrowserStreamIterator(stream, options) {
+    // WhatWG: stream is supposed to have a `getIterator` method
+    // if (typeof stream.getIterator === 'function') {
+    //   return stream.getIterator();
+    // }
+    // if (typeof stream[Symbol.asyncIterator] === 'function') {
+    //   return makeToArrayBufferIterator(stream);
+    // }
+    // In the browser, we first need to get a lock on the stream
+    const reader = stream.getReader();
+    let nextBatchPromise;
+    try {
+        // eslint-disable-next-line no-constant-condition
+        while (true) {
+            const currentBatchPromise = nextBatchPromise || reader.read();
+            // Issue a read for an additional batch, while we await the next batch
+            // Idea is to make fetching happen in parallel with processing / parsing
+            if (options?._streamReadAhead) {
+                nextBatchPromise = reader.read();
+            }
+            // Read from the stream
+            // value is a Uint8Array
+            const { done, value } = await currentBatchPromise;
+            // Exit if we're done
+            if (done) {
+                return;
+            }
+            // Else yield the chunk
+            if (value) {
+                yield value;
+            }
+        }
+    }
+    catch (error) {
+        // TODO - examples makes it look like this should always be called,
+        // but that generates exceptions so only call it if we do not reach the end
+        reader.releaseLock();
+    }
+}
+/**
+ * Returns an async iterable that reads from a DOM (browser) stream
+ * @param stream stream to iterate from
+ * @note Requires Node.js >= 10
+ */
+async function* makeNodeStreamIterator(stream, options) {
+    // Hacky test for node version to ensure we don't call bad polyfills
+    // NODE 10+: stream is an asyncIterator
+    yield* stream;
+}

package/dist/parquet-loader.js

@@ -8,7 +8,7 @@ import { parseParquetFileInColumns, parseParquetFileInColumnarBatches } from "./
 export { Buffer } from "./polyfills/buffer/install-buffer-polyfill.js";
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0" !== 'undefined' ? "4.2.0" : 'latest';
+const VERSION = typeof "4.3.0-alpha.1" !== 'undefined' ? "4.3.0-alpha.1" : 'latest';
 /**
  * ParquetJS table loader
  */

package/dist/parquet-wasm-loader.d.ts

@@ -1,16 +1,22 @@
+import type { ArrowTable, ArrowTableBatch } from '@loaders.gl/schema';
 import type { LoaderOptions } from '@loaders.gl/loader-utils';
-import type { ArrowTable } from '@loaders.gl/arrow';
 /** Parquet WASM loader options */
 export type ParquetWasmLoaderOptions = LoaderOptions & {
     parquet?: {
-        type?: 'arrow-table';
+        shape?: 'arrow-table';
+        limit?: number;
+        offset?: number;
+        batchSize?: number;
+        columns?: string[];
+        rowGroups?: number[];
+        concurrency?: number;
         wasmUrl?: string;
     };
 };
 /** Parquet WASM table loader */
 export declare const ParquetWasmWorkerLoader: {
     readonly dataType: ArrowTable;
-    readonly batchType: never;
+    readonly batchType: ArrowTableBatch;
     readonly name: "Apache Parquet";
     readonly id: "parquet-wasm";
     readonly module: "parquet";
@@ -23,8 +29,14 @@ export declare const ParquetWasmWorkerLoader: {
     readonly tests: ["PAR1", "PARE"];
     readonly options: {
         readonly parquet: {
-            readonly type: "arrow-table";
-            readonly wasmUrl: "https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm";
+            readonly shape: "arrow-table";
+            readonly limit: undefined;
+            readonly offset: 0;
+            readonly batchSize: undefined;
+            readonly columns: undefined;
+            readonly rowGroups: undefined;
+            readonly concurrency: undefined;
+            readonly wasmUrl: "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
         };
     };
 };

package/dist/parquet-wasm-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACtF,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,kCAAkC;AAClC,MAAM,MAAM,wBAAwB,GAAG,aAAa,GAAG;IACrD,OAAO,CAAC,EAAE;QACR,IAAI,CAAC,EAAE,aAAa,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;CAoBoC,CAAC;AAEzE,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,KAMoD,CAAC"}
+{"version":3,"file":"parquet-wasm-loader.d.ts","sourceRoot":"","sources":["../src/parquet-wasm-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAE,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACpE,OAAO,KAAK,EAA2B,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAStF,kCAAkC;AAClC,MAAM,MAAM,wBAAwB,GAAG,aAAa,GAAG;IACrD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,aAAa,CAAC;QACtB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;QACnB,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB,CAAC;CACH,CAAC;AAEF,gCAAgC;AAChC,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;;;;;;;;;;;;CA0B8C,CAAC;AAEnF,gCAAgC;AAChC,eAAO,MAAM,iBAAiB,KAiB8D,CAAC"}

package/dist/parquet-wasm-loader.js

@@ -1,7 +1,8 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { parseParquetWasm } from "./lib/wasm/parse-parquet-wasm.js";
+import { BlobFile } from '@loaders.gl/loader-utils';
+import { parseParquetFileWasm, parseParquetFileInBatchesWasm } from "./lib/parsers/parse-parquet-wasm.js";
 import { VERSION, PARQUET_WASM_URL } from "./lib/constants.js";
 /** Parquet WASM table loader */
 export const ParquetWasmWorkerLoader = {
@@ -19,7 +20,13 @@ export const ParquetWasmWorkerLoader = {
     tests: ['PAR1', 'PARE'],
     options: {
         parquet: {
-            type: 'arrow-table',
+            shape: 'arrow-table',
+            limit: undefined, // Provide a limit to the number of rows to be read.
+            offset: 0, // Provide an offset to skip over the given number of rows.
+            batchSize: undefined, // The number of rows in each batch. If not provided, the upstream parquet default is 1024.
+            columns: undefined, // The column names from the file to read.
+            rowGroups: undefined, // Only read data from the provided row group indexes.
+            concurrency: undefined, // The number of concurrent requests to make
             wasmUrl: PARQUET_WASM_URL
         }
     }
@@ -28,7 +35,15 @@ export const ParquetWasmWorkerLoader = {
 export const ParquetWasmLoader = {
     ...ParquetWasmWorkerLoader,
     parse(arrayBuffer, options) {
-        options = { parquet: { ...ParquetWasmLoader.options.parquet, ...options?.parquet }, ...options };
-        return parseParquetWasm(arrayBuffer, options);
+        const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options?.parquet };
+        return parseParquetFileWasm(new BlobFile(arrayBuffer), wasmOptions);
+    },
+    parseFile(file, options) {
+        const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options?.parquet };
+        return parseParquetFileWasm(file, wasmOptions);
+    },
+    parseFileInBatches(file, options) {
+        const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options?.parquet };
+        return parseParquetFileInBatchesWasm(file, wasmOptions);
     }
 };

package/dist/parquet-wasm-writer.js

@@ -1,7 +1,7 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import { encode } from "./lib/wasm/encode-parquet-wasm.js";
+import { encode } from "./lib/encoders/encode-parquet-wasm.js";
 import { VERSION, PARQUET_WASM_URL } from "./lib/constants.js";
 /** Parquet WASM writer */
 export const ParquetWasmWriter = {

package/dist/parquet-writer.js

@@ -3,7 +3,7 @@
 // Copyright (c) vis.gl contributors
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof "4.2.0" !== 'undefined' ? "4.2.0" : 'latest';
+const VERSION = typeof "4.3.0-alpha.1" !== 'undefined' ? "4.3.0-alpha.1" : 'latest';
 export const ParquetWriter = {
     name: 'Apache Parquet',
     id: 'parquet',

package/dist/parquetjs/codecs/rle.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CAgCV"}
+{"version":3,"file":"rle.d.ts","sourceRoot":"","sources":["../../../src/parquetjs/codecs/rle.ts"],"names":[],"mappings":";AAEA,OAAO,KAAK,EAAC,aAAa,EAAC,6BAA0B;AACrD,OAAO,KAAK,EAAC,YAAY,EAAE,mBAAmB,EAAC,qBAAkB;AAIjE,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,GAAG,EAAE,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,CA4DR;AAED,wBAAgB,YAAY,CAC1B,IAAI,EAAE,aAAa,EACnB,MAAM,EAAE,YAAY,EACpB,KAAK,EAAE,MAAM,EACb,IAAI,EAAE,mBAAmB,GACxB,MAAM,EAAE,CAkCV"}

package/dist/parquetjs/codecs/rle.js

@@ -76,6 +76,7 @@ export function decodeValues(type, cursor, count, opts) {
             const count = header >> 1;
             decodedValues = decodeRunRepeated(cursor, count, opts);
         }
+        // strange failure in docusaurus / webpack if we don't cast the type here
         for (const value of decodedValues) {
             values.push(value);
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.1",
+  "version": "4.3.0-alpha.2",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -43,7 +43,7 @@
     "build-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.min.js",
     "build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js",
     "build-worker": "esbuild src/workers/parquet-worker.ts --outfile=dist/parquet-worker.js --bundle --minify --sourcemap --external:{util,fs,path,crypto,events} --define:__VERSION__=\\\"$npm_package_version\\\"",
-    "copy-wasm": "cp ../../node_modules/parquet-wasm/esm2/arrow1_bg.wasm dist/arrow1_bg.wasm"
+    "copy-wasm": "cp ../../node_modules/parquet-wasm/esm/parquet_wasm_bg.wasm dist/parquet_wasm_bg.wasm"
   },
   "browser": {
     "./dist/polyfills/buffer/buffer-polyfill.node.js": "./dist/polyfills/buffer/buffer-polyfill.browser.js",
@@ -60,13 +60,14 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.1",
-    "@loaders.gl/bson": "4.2.1",
-    "@loaders.gl/compression": "4.2.1",
-    "@loaders.gl/gis": "4.2.1",
-    "@loaders.gl/loader-utils": "4.2.1",
-    "@loaders.gl/schema": "4.2.1",
-    "@loaders.gl/wkt": "4.2.1",
+    "@loaders.gl/arrow": "4.3.0-alpha.2",
+    "@loaders.gl/bson": "4.3.0-alpha.2",
+    "@loaders.gl/compression": "4.3.0-alpha.2",
+    "@loaders.gl/gis": "4.3.0-alpha.2",
+    "@loaders.gl/loader-utils": "4.3.0-alpha.2",
+    "@loaders.gl/schema": "4.3.0-alpha.2",
+    "@loaders.gl/wkt": "4.3.0-alpha.2",
+    "@probe.gl/log": "^4.0.9",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -75,11 +76,11 @@
     "lz4js": "^0.2.0",
     "node-int64": "^0.4.0",
     "object-stream": "0.0.1",
-    "parquet-wasm": "^0.6.0-beta.1",
+    "parquet-wasm": "^0.6.1",
     "snappyjs": "^0.6.0",
     "thrift": "^0.19.0",
     "util": "^0.12.5",
-    "varint": "^5.0.0",
+    "varint": "^6.0.0",
     "zstd-codec": "^0.1"
   },
   "devDependencies": {
@@ -93,5 +94,5 @@
     "@loaders.gl/core": "^4.0.0",
     "apache-arrow": ">= 15.0.0"
   },
-  "gitHead": "b2ea799ff9f233ab8af0a6c3696e28ddf6e3c5a3"
+  "gitHead": "77a3cb538ab7a1fbf74245f25590210451689f5c"
 }

package/src/lib/constants.ts

@@ -5,7 +5,8 @@
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 export const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';
+
+export const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm';
 
 /**
  * Parquet File Magic String

package/src/lib/{wasm → encoders}/encode-parquet-wasm.ts

@@ -5,7 +5,7 @@
 import type {ArrowTable} from '@loaders.gl/arrow';
 
 import * as arrow from 'apache-arrow';
-import {loadWasm} from './load-wasm';
+import {loadWasm} from '../utils/load-wasm';
 
 import type {ParquetWriterOptions} from '../../parquet-wasm-writer';
 

package/src/lib/parsers/parse-parquet-wasm.ts

@@ -0,0 +1,72 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+// eslint-disable
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import type {ArrowTable, ArrowTableBatch, Schema} from '@loaders.gl/schema';
+import {serializeArrowSchema} from '@loaders.gl/arrow';
+
+import type * as parquetWasm from 'parquet-wasm';
+import * as arrow from 'apache-arrow';
+
+import {loadWasm} from '../utils/load-wasm';
+import {makeStreamIterator} from '../utils/make-stream-iterator';
+
+export async function parseParquetFileWasm(
+  file: ReadableFile,
+  options?: parquetWasm.ReaderOptions & {wasmUrl: string}
+): Promise<ArrowTable> {
+  const wasmUrl = options?.wasmUrl;
+
+  const wasm = await loadWasm(wasmUrl);
+
+  let parquetFile: parquetWasm.ParquetFile;
+  if (file.handle instanceof Blob) {
+    // TODO - let's assume fromFile() works on Blobs and not just on File...
+    parquetFile = await wasm.ParquetFile.fromFile(file.handle as File);
+  } else {
+    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+  }
+
+  const wasmTable = await parquetFile.read(options);
+  const ipcStream = wasmTable.intoIPCStream();
+  const arrowTable = arrow.tableFromIPC(ipcStream);
+
+  return {
+    shape: 'arrow-table',
+    schema: serializeArrowSchema(arrowTable.schema),
+    data: arrowTable
+  };
+}
+
+export async function* parseParquetFileInBatchesWasm(
+  file: ReadableFile,
+  options: parquetWasm.ReaderOptions & {wasmUrl: string}
+): AsyncIterable<ArrowTableBatch> {
+  const wasmUrl = options?.wasmUrl;
+
+  const wasm = await loadWasm(wasmUrl);
+
+  let parquetFile: parquetWasm.ParquetFile;
+  if (file.handle instanceof Blob) {
+    // Works on Blobs: https://kylebarron.dev/parquet-wasm/classes/esm_parquet_wasm.ParquetFile.html#fromFile
+    parquetFile = await wasm.ParquetFile.fromFile(file.handle as File);
+  } else {
+    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+  }
+
+  const stream: ReadableStream<arrow.Table> = await parquetFile.stream(options);
+
+  let schema: Schema;
+  for await (const table of makeStreamIterator(stream)) {
+    schema ||= serializeArrowSchema(table.schema);
+    yield {
+      batchType: 'data',
+      shape: 'arrow-table',
+      schema,
+      data: table.batches[0],
+      length: table.numRows
+    };
+  }
+}

package/src/lib/parsers/parse-parquet.ts

@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import {default as log} from '@probe.gl/log';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
 
@@ -35,10 +36,19 @@ export async function parseParquetFile(
 
   const rowBatches = reader.rowBatchIterator(options?.parquet);
   for await (const rowBatch of rowBatches) {
+    let limitHasReached = false;
     // we have only one input batch so return
     for (const row of rowBatch) {
+      if (options?.limit && rows.length >= options?.limit) {
+        limitHasReached = true;
+        break;
+      }
       rows.push(row);
     }
+    if (limitHasReached) {
+      log.warn(`Rows number limit has been reached. Only first ${options?.limit} are loaded`)();
+      break;
+    }
   }
   const objectRowTable: ObjectRowTable = {
     shape: 'object-row-table',

package/src/lib/utils/make-stream-iterator.ts

@@ -0,0 +1,87 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {Readable} from 'stream';
+import {isBrowser} from '@loaders.gl/loader-utils';
+
+export type StreamIteratorOptions = {
+  _streamReadAhead?: boolean;
+};
+
+/**
+ * Returns an async iterable that reads from a stream (works in both Node.js and browsers)
+ * @param stream stream to iterator over
+ */
+export function makeStreamIterator<T>(
+  stream: ReadableStream<T> | Readable,
+  options?: StreamIteratorOptions
+): AsyncIterable<T> {
+  return isBrowser
+    ? makeBrowserStreamIterator(stream as ReadableStream<T>, options)
+    : makeNodeStreamIterator(stream as Readable, options);
+}
+
+/**
+ * Returns an async iterable that reads from a DOM (browser) stream
+ * @param stream stream to iterate from
+ * @see https://jakearchibald.com/2017/async-iterators-and-generators/#making-streams-iterate
+ */
+async function* makeBrowserStreamIterator<T>(
+  stream: ReadableStream<T>,
+  options?: StreamIteratorOptions
+): AsyncIterable<T> {
+  // WhatWG: stream is supposed to have a `getIterator` method
+  // if (typeof stream.getIterator === 'function') {
+  //   return stream.getIterator();
+  // }
+  // if (typeof stream[Symbol.asyncIterator] === 'function') {
+  //   return makeToArrayBufferIterator(stream);
+  // }
+
+  // In the browser, we first need to get a lock on the stream
+  const reader = stream.getReader();
+
+  let nextBatchPromise: Promise<{done?: boolean; value?: T}> | undefined;
+
+  try {
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+      const currentBatchPromise = nextBatchPromise || reader.read();
+      // Issue a read for an additional batch, while we await the next batch
+      // Idea is to make fetching happen in parallel with processing / parsing
+      if (options?._streamReadAhead) {
+        nextBatchPromise = reader.read();
+      }
+      // Read from the stream
+      // value is a Uint8Array
+      const {done, value} = await currentBatchPromise;
+      // Exit if we're done
+      if (done) {
+        return;
+      }
+      // Else yield the chunk
+      if (value) {
+        yield value;
+      }
+    }
+  } catch (error) {
+    // TODO - examples makes it look like this should always be called,
+    // but that generates exceptions so only call it if we do not reach the end
+    reader.releaseLock();
+  }
+}
+
+/**
+ * Returns an async iterable that reads from a DOM (browser) stream
+ * @param stream stream to iterate from
+ * @note Requires Node.js >= 10
+ */
+async function* makeNodeStreamIterator<T>(
+  stream: Readable,
+  options?: StreamIteratorOptions
+): AsyncIterable<T> {
+  // Hacky test for node version to ensure we don't call bad polyfills
+  // NODE 10+: stream is an asyncIterator
+  yield* stream;
+}