@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

package/src/lib/parsers/get-parquet-schema.ts

@@ -4,14 +4,16 @@
 
 // loaders.gl
 import {Schema} from '@loaders.gl/schema';
-import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/geoarrow';
+import {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/gis';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {convertParquetSchema} from '../arrow/convert-schema-from-parquet';
+import {applyGeoParquetToFieldMetadata} from '../geo/geospatial-metadata';
 
 export async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {
   const parquetSchema = await reader.getSchema();
   const parquetMetadata = await reader.getFileMetadata();
   const schema = convertParquetSchema(parquetSchema, parquetMetadata);
+  applyGeoParquetToFieldMetadata(schema);
   unpackGeoMetadata(schema.metadata);
   unpackJSONStringMetadata(schema.metadata, 'pandas');
   return schema;

package/src/lib/parsers/parse-geoparquet-to-geojson.ts

@@ -6,31 +6,44 @@ import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {
   GeoJSONTable,
   GeoJSONTableBatch,
-  ObjectRowTable,
-  ObjectRowTableBatch
+  ArrowTableBatch,
+  ObjectRowTable
 } from '@loaders.gl/schema';
 import {convertWKBTableToGeoJSON} from '@loaders.gl/gis';
+import {convertArrowToTable} from '@loaders.gl/schema-utils';
 
-import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
-
-import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet-to-json';
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
+import {parseParquetArrowTable, parseParquetArrowTableInBatches} from './parse-parquet-tables';
 
 export async function parseGeoParquetFile(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
-): Promise<ObjectRowTable | GeoJSONTable> {
-  const table = await parseParquetFile(file, {...options, shape: 'object-row-table'});
-  return convertWKBTableToGeoJSON(table, table.schema!);
+  options?: ParquetLoaderOptions
+): Promise<GeoJSONTable> {
+  const arrowTable = await parseParquetArrowTable(file, options || {});
+  const objectRowTable = convertArrowToTable(arrowTable.data, 'object-row-table') as ObjectRowTable;
+  return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!);
 }
 
 export async function* parseGeoParquetFileInBatches(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
-): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {
-  const tableBatches = parseParquetFileInBatches(file, {...options, shape: 'object-row-table'});
+  options?: ParquetLoaderOptions
+): AsyncIterable<GeoJSONTableBatch> {
+  const tableBatches = parseParquetArrowTableInBatches(file, options || {});
 
   for await (const batch of tableBatches) {
-    const geojson = convertWKBTableToGeoJSON(batch as ObjectRowTable, batch.schema!);
-    yield {...batch, ...geojson};
+    yield convertGeoParquetArrowBatch(batch);
   }
 }
+
+function convertGeoParquetArrowBatch(batch: ArrowTableBatch): GeoJSONTableBatch {
+  const objectRowTable = convertArrowToTable(batch.data, 'object-row-table') as ObjectRowTable;
+  const geojsonTable = convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!);
+  return {
+    batchType: batch.batchType,
+    shape: geojsonTable.shape,
+    type: geojsonTable.type,
+    schema: geojsonTable.schema,
+    features: geojsonTable.features,
+    length: batch.length
+  };
+}

package/src/lib/parsers/parse-parquet-tables.ts

@@ -0,0 +1,137 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import {convertArrowToTable} from '@loaders.gl/schema-utils';
+import type {
+  ArrowTable,
+  ArrowTableBatch,
+  ObjectRowTable,
+  ObjectRowTableBatch
+} from '@loaders.gl/schema';
+
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
+import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet-to-json';
+import {parseParquetFileToArrow, parseParquetFileToArrowInBatches} from './parse-parquet-to-arrow';
+import {
+  parseParquetFileToArrowWithJs,
+  parseParquetFileToArrowInBatchesWithJs
+} from './parse-parquet-to-arrow-js';
+
+/**
+ * Parses a parquet file into an Arrow-backed table.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Arrow table output.
+ */
+export function parseParquetArrowTable(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): Promise<ArrowTable> {
+  switch (getParquetBackend(options)) {
+    case 'typescript':
+      return parseParquetFileToArrowWithJs(file, options);
+
+    case 'wasm':
+    default:
+      return parseParquetFileToArrow(file, options.parquet);
+  }
+}
+
+/**
+ * Parses a parquet file into Arrow-backed table batches.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Async iterable of Arrow table batches.
+ */
+export function parseParquetArrowTableInBatches(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): AsyncIterable<ArrowTableBatch> {
+  switch (getParquetBackend(options)) {
+    case 'typescript':
+      return parseParquetFileToArrowInBatchesWithJs(file, options);
+
+    case 'wasm':
+    default:
+      return parseParquetFileToArrowInBatches(file, options.parquet);
+  }
+}
+
+/**
+ * Parses a parquet file into plain object rows.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Object-row table output.
+ */
+export async function parseParquetObjectRowTable(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): Promise<ObjectRowTable> {
+  if (getParquetBackend(options) === 'typescript') {
+    return await parseParquetFile(file, options);
+  }
+
+  const arrowTable = await parseParquetArrowTable(file, options);
+  return convertArrowTableToObjectRows(arrowTable);
+}
+
+/**
+ * Parses a parquet file into object-row batches.
+ *
+ * @param file - Parquet file abstraction.
+ * @param options - Normalized parquet options.
+ * @returns Async iterable of object-row table batches.
+ */
+export async function* parseParquetObjectRowTableInBatches(
+  file: ReadableFile,
+  options: ParquetLoaderOptions
+): AsyncIterable<ObjectRowTableBatch> {
+  if (getParquetBackend(options) === 'typescript') {
+    yield* parseParquetFileInBatches(file, options);
+    return;
+  }
+
+  for await (const batch of parseParquetArrowTableInBatches(file, options)) {
+    yield convertArrowBatchToObjectRows(batch);
+  }
+}
+
+function getParquetBackend(options: ParquetLoaderOptions): 'wasm' | 'typescript' {
+  if (options.parquet?.backend) {
+    return options.parquet.backend;
+  }
+  return options.parquet?.implementation === 'js' ? 'typescript' : 'wasm';
+}
+
+/**
+ * Converts an Arrow-backed table into object rows.
+ *
+ * @param arrowTable - Arrow table wrapper.
+ * @returns Object-row table.
+ */
+export function convertArrowTableToObjectRows(arrowTable: ArrowTable): ObjectRowTable {
+  return convertArrowToTable(arrowTable.data, 'object-row-table') as ObjectRowTable;
+}
+
+/**
+ * Converts an Arrow batch into object-row output.
+ *
+ * @param batch - Arrow table batch wrapper.
+ * @returns Object-row batch.
+ */
+export function convertArrowBatchToObjectRows(batch: ArrowTableBatch): ObjectRowTableBatch {
+  const objectRowTable = convertArrowToTable(batch.data, 'object-row-table') as ObjectRowTable;
+
+  return {
+    batchType: batch.batchType,
+    shape: objectRowTable.shape,
+    schema: objectRowTable.schema,
+    data: objectRowTable.data,
+    length: batch.length
+  };
+}

package/src/lib/parsers/parse-parquet-to-arrow-js.ts

@@ -0,0 +1,36 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+import type {ReadableFile} from '@loaders.gl/loader-utils';
+import type {ArrowTable, ArrowTableBatch} from '@loaders.gl/schema';
+import {convertTable} from '@loaders.gl/schema-utils';
+
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
+import {normalizeArrowTableGeoMetadata} from '../geo/geospatial-metadata';
+import {parseParquetFile, parseParquetFileInBatches} from './parse-parquet-to-json';
+
+export async function parseParquetFileToArrowWithJs(
+  file: ReadableFile,
+  options?: ParquetLoaderOptions
+): Promise<ArrowTable> {
+  const objectRowTable = await parseParquetFile(file, options);
+  return normalizeArrowTableGeoMetadata(convertTable(objectRowTable, 'arrow-table'));
+}
+
+export async function* parseParquetFileToArrowInBatchesWithJs(
+  file: ReadableFile,
+  options?: ParquetLoaderOptions
+): AsyncIterable<ArrowTableBatch> {
+  for await (const batch of parseParquetFileInBatches(file, options)) {
+    const arrowTable = normalizeArrowTableGeoMetadata(convertTable(batch, 'arrow-table'));
+
+    yield {
+      batchType: batch.batchType,
+      shape: arrowTable.shape,
+      schema: arrowTable.schema,
+      data: arrowTable.data,
+      length: batch.length
+    };
+  }
+}

package/src/lib/parsers/parse-parquet-to-arrow.ts

@@ -5,68 +5,95 @@
 // eslint-disable
 import type {ReadableFile} from '@loaders.gl/loader-utils';
 import type {ArrowTable, ArrowTableBatch, Schema} from '@loaders.gl/schema';
-import {convertArrowToSchema} from '@loaders.gl/schema-utils';
 
-import type * as parquetWasm from 'parquet-wasm';
+import type * as parquetWasm from 'parquet-wasm/esm/parquet_wasm.js';
 import * as arrow from 'apache-arrow';
 
+import {
+  normalizeArrowTableGeoMetadata,
+  getParquetFileMetadataMap
+} from '../geo/geospatial-metadata';
 import {loadWasm} from '../utils/load-wasm';
 import {makeStreamIterator} from '../utils/make-stream-iterator';
 
+type ParquetWasmReaderOptions = parquetWasm.ReaderOptions & {wasmUrl?: string};
+
 export async function parseParquetFileToArrow(
   file: ReadableFile,
-  options?: parquetWasm.ReaderOptions & {wasmUrl: string}
+  options?: ParquetWasmReaderOptions
 ): Promise<ArrowTable> {
   const wasmUrl = options?.wasmUrl;
+  const readerOptions = getReaderOptions(options);
 
   const wasm = await loadWasm(wasmUrl);
 
-  let parquetFile: parquetWasm.ParquetFile;
-  if (file.handle instanceof Blob) {
-    // TODO - let's assume fromFile() works on Blobs and not just on File...
-    parquetFile = await wasm.ParquetFile.fromFile(file.handle as File);
-  } else {
-    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+  const parquetFile = await createParquetFile(file, wasm);
+  try {
+    const wasmTable = await parquetFile.read(readerOptions);
+    const ipcStream = wasmTable.intoIPCStream();
+    const arrowTable = arrow.tableFromIPC(ipcStream);
+    return normalizeArrowTableGeoMetadata(
+      {
+        shape: 'arrow-table',
+        data: arrowTable
+      },
+      getParquetFileMetadataMap(parquetFile.metadata())
+    );
+  } finally {
+    parquetFile.free();
   }
-
-  const wasmTable = await parquetFile.read(options);
-  const ipcStream = wasmTable.intoIPCStream();
-  const arrowTable = arrow.tableFromIPC(ipcStream);
-
-  return {
-    shape: 'arrow-table',
-    schema: convertArrowToSchema(arrowTable.schema),
-    data: arrowTable
-  };
 }
 
 export async function* parseParquetFileToArrowInBatches(
   file: ReadableFile,
-  options: parquetWasm.ReaderOptions & {wasmUrl: string}
+  options?: ParquetWasmReaderOptions
 ): AsyncIterable<ArrowTableBatch> {
   const wasmUrl = options?.wasmUrl;
+  const readerOptions = getReaderOptions(options);
 
   const wasm = await loadWasm(wasmUrl);
 
-  let parquetFile: parquetWasm.ParquetFile;
+  const parquetFile = await createParquetFile(file, wasm);
+  try {
+    const stream = await parquetFile.stream(readerOptions);
+    const schemaMetadata = getParquetFileMetadataMap(parquetFile.metadata());
+
+    let schema: Schema | undefined;
+    for await (const wasmRecordBatch of makeStreamIterator<parquetWasm.RecordBatch>(stream)) {
+      const normalizedArrowTable = normalizeArrowTableGeoMetadata(
+        {
+          shape: 'arrow-table',
+          data: arrow.tableFromIPC(wasmRecordBatch.intoIPCStream())
+        },
+        schemaMetadata
+      );
+      schema ||= normalizedArrowTable.schema;
+      yield {
+        batchType: 'data',
+        shape: 'arrow-table',
+        schema,
+        data: normalizedArrowTable.data,
+        length: normalizedArrowTable.data.numRows
+      };
+    }
+  } finally {
+    parquetFile.free();
+  }
+}
+
+async function createParquetFile(
+  file: ReadableFile,
+  wasm: typeof parquetWasm
+): Promise<parquetWasm.ParquetFile> {
   if (file.handle instanceof Blob) {
     // Works on Blobs: https://kylebarron.dev/parquet-wasm/classes/esm_parquet_wasm.ParquetFile.html#fromFile
-    parquetFile = await wasm.ParquetFile.fromFile(file.handle as File);
-  } else {
-    parquetFile = await wasm.ParquetFile.fromUrl(file.url);
+    return await wasm.ParquetFile.fromFile(file.handle as File);
   }
 
-  const stream: ReadableStream<arrow.RecordBatch> = await parquetFile.stream(options);
+  return await wasm.ParquetFile.fromUrl(file.url);
+}
 
-  let schema: Schema;
-  for await (const recordBatch of makeStreamIterator(stream)) {
-    schema ||= convertArrowToSchema(recordBatch.schema);
-    yield {
-      batchType: 'data',
-      shape: 'arrow-table',
-      schema,
-      data: new arrow.Table(recordBatch),
-      length: recordBatch.numRows
-    };
-  }
+function getReaderOptions(options?: ParquetWasmReaderOptions): parquetWasm.ReaderOptions {
+  const {wasmUrl, ...readerOptions} = options || {};
+  return readerOptions;
 }

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -4,13 +4,12 @@
 
 import type {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
-import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
+import type {ParquetLoaderOptions} from '../../parquet-loader';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {ParquetRowGroup} from '../../parquetjs/schema/declare';
 import {ParquetSchema} from '../../parquetjs/schema/schema';
 import {materializeColumns} from '../../parquetjs/schema/shred';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
-import {installBufferPolyfill} from '../../polyfills/buffer/index';
 import {preloadCompressions} from '../../parquetjs/compression';
 
 /**
@@ -18,9 +17,8 @@ import {preloadCompressions} from '../../parquetjs/compression';
  */
 export async function parseParquetFileInColumns(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
+  options?: ParquetLoaderOptions
 ): Promise<ColumnarTable> {
-  installBufferPolyfill();
   await preloadCompressions(options);
 
   for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
@@ -38,9 +36,8 @@ export async function parseParquetFileInColumns(
  */
 export async function* parseParquetFileInColumnarBatches(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
+  options?: ParquetLoaderOptions
 ): AsyncIterable<ColumnarTableBatch> {
-  installBufferPolyfill();
   await preloadCompressions(options);
 
   const reader = new ParquetReader(file);
@@ -51,7 +48,9 @@ export async function* parseParquetFileInColumnarBatches(
   const parquetSchema = await reader.getSchema();
 
   // Iterate over row batches
-  const rowGroups = reader.rowGroupIterator(options?.parquet);
+  const rowGroups = reader.rowGroupIterator(
+    options?.parquet?.columns?.length ? {columnList: options.parquet.columns} : undefined
+  );
   for await (const rowGroup of rowGroups) {
     yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);
   }

package/src/lib/parsers/parse-parquet-to-json.ts

@@ -4,13 +4,12 @@
 
 import {default as log} from '@probe.gl/log';
 import type {ReadableFile} from '@loaders.gl/loader-utils';
-import type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';
+import type {ObjectRowTable, ObjectRowTableBatch, Schema} from '@loaders.gl/schema';
 
-import type {ParquetJSONLoaderOptions} from '../../parquet-json-loader';
+import type {ParquetLoaderOptions} from '../../parquet-loader-options';
 import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
-import {installBufferPolyfill} from '../../polyfills/buffer/index';
 import {preloadCompressions} from '../../parquetjs/compression';
 
 /**
@@ -21,9 +20,8 @@ import {preloadCompressions} from '../../parquetjs/compression';
  */
 export async function parseParquetFile(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
+  options?: ParquetLoaderOptions
 ): Promise<ObjectRowTable> {
-  installBufferPolyfill();
   await preloadCompressions(options);
 
   const reader = new ParquetReader(file, {
@@ -31,30 +29,37 @@ export async function parseParquetFile(
   });
 
   const schema = await getSchemaFromParquetReader(reader);
+  const projectedSchema = projectSchema(schema, options?.parquet?.columns);
 
   const rows: ParquetRow[] = [];
 
-  const rowBatches = reader.rowBatchIterator(options?.parquet);
+  const rowBatches = reader.rowBatchIterator(getParquetIterationProps(options));
+  const rowOffset = options?.parquet?.offset || 0;
+  const rowLimit = options?.parquet?.limit;
+  let rowsSkipped = 0;
   for await (const rowBatch of rowBatches) {
-    let limitHasReached = false;
-    // we have only one input batch so return
     for (const row of rowBatch) {
-      if (options?.limit && rows.length >= options?.core?.limit!) {
-        limitHasReached = true;
-        break;
+      if (rowsSkipped < rowOffset) {
+        rowsSkipped++;
+        continue;
+      }
+
+      if (rowLimit !== undefined && rows.length >= rowLimit) {
+        log.warn(`Rows number limit has been reached. Only first ${rowLimit} are loaded`)();
+        return {
+          shape: 'object-row-table',
+          schema: projectedSchema,
+          data: rows
+        };
       }
+
       rows.push(row);
     }
-    if (limitHasReached) {
-      log.warn(
-        `Rows number limit has been reached. Only first ${options?.core?.limit} are loaded`
-      )();
-      break;
-    }
   }
+
   const objectRowTable: ObjectRowTable = {
     shape: 'object-row-table',
-    schema,
+    schema: projectedSchema,
     data: rows
   };
 
@@ -68,9 +73,8 @@ export async function parseParquetFile(
  */
 export async function* parseParquetFileInBatches(
   file: ReadableFile,
-  options?: ParquetJSONLoaderOptions
+  options?: ParquetLoaderOptions
 ): AsyncIterable<ObjectRowTableBatch> {
-  installBufferPolyfill();
   await preloadCompressions(options);
 
   const reader = new ParquetReader(file, {
@@ -78,19 +82,83 @@ export async function* parseParquetFileInBatches(
   });
 
   const schema = await getSchemaFromParquetReader(reader);
-  const rowBatches = reader.rowBatchIterator(options?.parquet);
+  const projectedSchema = projectSchema(schema, options?.parquet?.columns);
+  const rowBatches = reader.rowBatchIterator(getParquetIterationProps(options));
+  const rowOffset = options?.parquet?.offset || 0;
+  const rowLimit = options?.parquet?.limit;
+  const outputBatchSize = options?.parquet?.batchSize;
+  let rowsSkipped = 0;
+  let rowsYielded = 0;
+  let reachedLimit = false;
+
   for await (const rows of rowBatches) {
-    const objectRowTable: ObjectRowTable = {
-      shape: 'object-row-table',
-      schema,
-      data: rows
-    };
-
-    yield {
-      batchType: 'data',
-      schema,
-      ...objectRowTable,
-      length: rows.length
-    };
+    const nextRows: ParquetRow[] = [];
+    for (const row of rows) {
+      if (rowsSkipped < rowOffset) {
+        rowsSkipped++;
+        continue;
+      }
+
+      if (rowLimit !== undefined && rowsYielded >= rowLimit) {
+        reachedLimit = true;
+        break;
+      }
+
+      nextRows.push(row);
+      rowsYielded++;
+    }
+
+    if (nextRows.length === 0) {
+      continue;
+    }
+
+    if (!outputBatchSize || outputBatchSize <= 0) {
+      yield makeObjectRowBatch(projectedSchema, nextRows);
+      continue;
+    }
+
+    for (let rowIndex = 0; rowIndex < nextRows.length; rowIndex += outputBatchSize) {
+      yield makeObjectRowBatch(
+        projectedSchema,
+        nextRows.slice(rowIndex, rowIndex + outputBatchSize)
+      );
+    }
+
+    if (reachedLimit) {
+      return;
+    }
   }
 }
+
+function getParquetIterationProps(
+  options?: ParquetLoaderOptions
+): {columnList?: string[] | string[][]} | undefined {
+  const columnList = options?.parquet?.columns;
+  return columnList?.length ? {columnList} : undefined;
+}
+
+function makeObjectRowBatch(schema: Schema, rows: ParquetRow[]): ObjectRowTableBatch {
+  const objectRowTable: ObjectRowTable = {
+    shape: 'object-row-table',
+    schema,
+    data: rows
+  };
+
+  return {
+    batchType: 'data',
+    schema,
+    ...objectRowTable,
+    length: rows.length
+  };
+}
+
+function projectSchema(schema: Schema, columns?: string[]): Schema {
+  if (!columns?.length) {
+    return schema;
+  }
+
+  return {
+    ...schema,
+    fields: schema.fields.filter(field => columns.includes(field.name))
+  };
+}

package/src/lib/utils/load-wasm.ts

@@ -2,21 +2,28 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-// eslint-disable-next-line import/default
-import initWasm from 'parquet-wasm';
-import * as parquetWasm from 'parquet-wasm';
+import type * as ParquetWasm from 'parquet-wasm/esm/parquet_wasm.js';
 import {PARQUET_WASM_URL} from '../constants';
 
-let initializePromise: any;
+let initializePromise: Promise<typeof ParquetWasm>;
 
-export async function loadWasm(wasmUrl: string = PARQUET_WASM_URL) {
-  if (!initializePromise && typeof initWasm === 'function') {
+export async function loadWasm(
+  wasmUrl: ParquetWasm.InitInput | Promise<ParquetWasm.InitInput> = PARQUET_WASM_URL
+): Promise<typeof ParquetWasm> {
+  if (!initializePromise) {
     if (!wasmUrl) {
       throw new Error('ParquetLoader: No wasmUrl provided');
     }
-    // @ts-ignore
-    initializePromise = initWasm(wasmUrl);
+    initializePromise = loadAndInitializeWasm(wasmUrl);
   }
-  await initializePromise;
+
+  return await initializePromise;
+}
+
+async function loadAndInitializeWasm(
+  wasmUrl: ParquetWasm.InitInput | Promise<ParquetWasm.InitInput>
+): Promise<typeof ParquetWasm> {
+  const parquetWasm = await import('parquet-wasm/esm/parquet_wasm.js');
+  await parquetWasm.default({module_or_path: wasmUrl});
   return parquetWasm;
 }

package/src/lib/utils/make-stream-iterator.ts

@@ -65,7 +65,7 @@ async function* makeBrowserStreamIterator<T>(
         yield value;
       }
     }
-  } catch (error) {
+  } catch (_error) {
     // TODO - examples makes it look like this should always be called,
     // but that generates exceptions so only call it if we do not reach the end
     reader.releaseLock();

package/src/lib/utils/normalize-parquet-options.ts

@@ -0,0 +1,14 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+
+/** Merge caller options with default parquet options while preserving top-level option bags. */
+export function normalizeParquetOptions<OptionsT extends {parquet?: unknown}>(
+  options: OptionsT | undefined,
+  parquetDefaults: Record<string, unknown>
+): OptionsT {
+  return {
+    ...options,
+    parquet: {...parquetDefaults, ...(options?.parquet as Record<string, unknown> | undefined)}
+  } as unknown as OptionsT;
+}