@loaders.gl/parquet 4.2.1 → 4.3.0-alpha.2

package/src/parquet-wasm-loader.ts

@@ -2,16 +2,26 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
 import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import type {ArrowTable} from '@loaders.gl/arrow';
+import {ReadableFile, BlobFile} from '@loaders.gl/loader-utils';
 
-import {parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
+import {
+  parseParquetFileWasm,
+  parseParquetFileInBatchesWasm
+} from './lib/parsers/parse-parquet-wasm';
 import {VERSION, PARQUET_WASM_URL} from './lib/constants';
 
 /** Parquet WASM loader options */
 export type ParquetWasmLoaderOptions = LoaderOptions & {
   parquet?: {
-    type?: 'arrow-table';
+    shape?: 'arrow-table';
+    limit?: number; // Provide a limit to the number of rows to be read.
+    offset?: number; // Provide an offset to skip over the given number of rows.
+    batchSize?: number; // The number of rows in each batch. If not provided, the upstream parquet default is 1024.
+    columns?: string[]; // The column names from the file to read.
+    rowGroups?: number[]; // Only read data from the provided row group indexes.
+    concurrency?: number; // The number of concurrent requests to make
     wasmUrl?: string;
   };
 };
@@ -19,7 +29,7 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
 /** Parquet WASM table loader */
 export const ParquetWasmWorkerLoader = {
   dataType: null as unknown as ArrowTable,
-  batchType: null as never,
+  batchType: null as unknown as ArrowTableBatch,
 
   name: 'Apache Parquet',
   id: 'parquet-wasm',
@@ -33,17 +43,34 @@ export const ParquetWasmWorkerLoader = {
   tests: ['PAR1', 'PARE'],
   options: {
     parquet: {
-      type: 'arrow-table',
+      shape: 'arrow-table',
+      limit: undefined, // Provide a limit to the number of rows to be read.
+      offset: 0, // Provide an offset to skip over the given number of rows.
+      batchSize: undefined, // The number of rows in each batch. If not provided, the upstream parquet default is 1024.
+      columns: undefined, // The column names from the file to read.
+      rowGroups: undefined, // Only read data from the provided row group indexes.
+      concurrency: undefined, // The number of concurrent requests to make
       wasmUrl: PARQUET_WASM_URL
     }
   }
-} as const satisfies Loader<ArrowTable, never, ParquetWasmLoaderOptions>;
+} as const satisfies Loader<ArrowTable, ArrowTableBatch, ParquetWasmLoaderOptions>;
 
 /** Parquet WASM table loader */
 export const ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
+
   parse(arrayBuffer: ArrayBuffer, options?: ParquetWasmLoaderOptions) {
-    options = {parquet: {...ParquetWasmLoader.options.parquet, ...options?.parquet}, ...options};
-    return parseParquetWasm(arrayBuffer, options);
+    const wasmOptions = {...ParquetWasmLoader.options.parquet, ...options?.parquet};
+    return parseParquetFileWasm(new BlobFile(arrayBuffer), wasmOptions);
+  },
+
+  parseFile(file: ReadableFile, options?: ParquetWasmLoaderOptions) {
+    const wasmOptions = {...ParquetWasmLoader.options.parquet, ...options?.parquet};
+    return parseParquetFileWasm(file, wasmOptions);
+  },
+
+  parseFileInBatches(file: ReadableFile, options?: ParquetWasmLoaderOptions) {
+    const wasmOptions = {...ParquetWasmLoader.options.parquet, ...options?.parquet};
+    return parseParquetFileInBatchesWasm(file, wasmOptions);
   }
-} as const satisfies LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions>;
+} as const satisfies LoaderWithParser<ArrowTable, ArrowTableBatch, ParquetWasmLoaderOptions>;

package/src/parquet-wasm-writer.ts

@@ -4,7 +4,7 @@
 
 import type {WriterWithEncoder} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/arrow';
-import {encode} from './lib/wasm/encode-parquet-wasm';
+import {encode} from './lib/encoders/encode-parquet-wasm';
 import type {WriterOptions} from '@loaders.gl/loader-utils';
 
 import {VERSION, PARQUET_WASM_URL} from './lib/constants';

package/src/parquetjs/codecs/rle.ts

@@ -97,7 +97,9 @@ export function decodeValues(
       const count = header >> 1;
       decodedValues = decodeRunRepeated(cursor, count, opts);
     }
-    for (const value of decodedValues) {
+
+    // strange failure in docusaurus / webpack if we don't cast the type here
+    for (const value of decodedValues as any[]) {
       values.push(value);
     }
   }

package/dist/lib/wasm/encode-parquet-wasm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"encode-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/encode-parquet-wasm.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAKlD,OAAO,KAAK,EAAC,oBAAoB,EAAC,qCAAkC;AAEpE;;GAEG;AACH,wBAAsB,MAAM,CAC1B,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,oBAAoB,GAC5B,OAAO,CAAC,WAAW,CAAC,CAsBtB"}

package/dist/lib/wasm/load-wasm.d.ts

@@ -1,3 +0,0 @@
-import initWasm from 'parquet-wasm';
-export declare function loadWasm(wasmUrl?: string): Promise<typeof initWasm>;
-//# sourceMappingURL=load-wasm.d.ts.map

package/dist/lib/wasm/load-wasm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"load-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/load-wasm.ts"],"names":[],"mappings":"AAKA,OAAO,QAAQ,MAAM,cAAc,CAAC;AAMpC,wBAAsB,QAAQ,CAAC,OAAO,GAAE,MAAyB,4BAUhE"}

package/dist/lib/wasm/parse-parquet-wasm.d.ts

@@ -1,4 +0,0 @@
-import type { ArrowTable } from '@loaders.gl/arrow';
-import type { ParquetWasmLoaderOptions } from "../../parquet-wasm-loader.js";
-export declare function parseParquetWasm(arrayBuffer: ArrayBuffer, options: ParquetWasmLoaderOptions): Promise<ArrowTable>;
-//# sourceMappingURL=parse-parquet-wasm.d.ts.map

package/dist/lib/wasm/parse-parquet-wasm.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-wasm.d.ts","sourceRoot":"","sources":["../../../src/lib/wasm/parse-parquet-wasm.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAElD,OAAO,KAAK,EAAC,wBAAwB,EAAC,qCAAkC;AAIxE,wBAAsB,gBAAgB,CACpC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,wBAAwB,GAChC,OAAO,CAAC,UAAU,CAAC,CAkBrB"}

package/dist/lib/wasm/parse-parquet-wasm.js

@@ -1,24 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-import { serializeArrowSchema } from '@loaders.gl/arrow';
-import { loadWasm } from "./load-wasm.js";
-import * as arrow from 'apache-arrow';
-export async function parseParquetWasm(arrayBuffer, options) {
-    const arr = new Uint8Array(arrayBuffer);
-    const wasmUrl = options?.parquet?.wasmUrl;
-    const wasm = await loadWasm(wasmUrl);
-    const wasmTable = wasm.readParquet(arr);
-    try {
-        const ipcStream = wasmTable.intoIPCStream();
-        const arrowTable = arrow.tableFromIPC(ipcStream);
-        return {
-            shape: 'arrow-table',
-            schema: serializeArrowSchema(arrowTable.schema),
-            data: arrowTable
-        };
-    }
-    finally {
-        // wasmTable.free();
-    }
-}

package/src/lib/wasm/parse-parquet-wasm.ts

@@ -1,33 +0,0 @@
-// loaders.gl
-// SPDX-License-Identifier: MIT
-// Copyright (c) vis.gl contributors
-
-// eslint-disable
-import type {ArrowTable} from '@loaders.gl/arrow';
-import {serializeArrowSchema} from '@loaders.gl/arrow';
-import type {ParquetWasmLoaderOptions} from '../../parquet-wasm-loader';
-import {loadWasm} from './load-wasm';
-import * as arrow from 'apache-arrow';
-
-export async function parseParquetWasm(
-  arrayBuffer: ArrayBuffer,
-  options: ParquetWasmLoaderOptions
-): Promise<ArrowTable> {
-  const arr = new Uint8Array(arrayBuffer);
-
-  const wasmUrl = options?.parquet?.wasmUrl;
-  const wasm = await loadWasm(wasmUrl);
-  const wasmTable = wasm.readParquet(arr);
-  try {
-    const ipcStream = wasmTable.intoIPCStream();
-    const arrowTable = arrow.tableFromIPC(ipcStream);
-
-    return {
-      shape: 'arrow-table',
-      schema: serializeArrowSchema(arrowTable.schema),
-      data: arrowTable
-    };
-  } finally {
-    // wasmTable.free();
-  }
-}