@loaders.gl/parquet 4.0.0-beta.1 → 4.0.0-beta.3

package/src/parquet-loader.ts

@@ -9,6 +9,8 @@ import type {
   GeoJSONTableBatch
 } from '@loaders.gl/schema';
 
+export {Buffer} from './buffer-polyfill/install-buffer-polyfill';
+
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
@@ -57,7 +59,7 @@ export const ParquetLoader: Loader<
   }
 };
 
-export const ParquetColumnnarLoader: Loader<
+export const ParquetColumnarLoader: Loader<
   ColumnarTable,
   ColumnarTableBatch,
   ParquetLoaderOptions
@@ -74,3 +76,9 @@ export const ParquetColumnnarLoader: Loader<
   tests: ['PAR1', 'PARE'],
   options: ParquetLoader.options
 };
+
+// Defeat tree shaking
+// @ts-ignore
+ParquetLoader.Buffer = Buffer;
+// @ts-ignore
+ParquetColumnarLoader.Buffer = Buffer;

package/src/parquetjs/parser/parquet-reader.ts

@@ -134,7 +134,8 @@ export class ParquetReader {
 
   /** Metadata is stored in the footer */
   async readHeader(): Promise<void> {
-    const buffer = await this.file.read(0, PARQUET_MAGIC.length);
+    const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);
+    const buffer = Buffer.from(arrayBuffer);
     const magic = buffer.toString();
     switch (magic) {
       case PARQUET_MAGIC:
@@ -149,7 +150,8 @@ export class ParquetReader {
   /** Metadata is stored in the footer */
   async readFooter(): Promise<FileMetaData> {
     const trailerLen = PARQUET_MAGIC.length + 4;
-    const trailerBuf = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);
+    const trailerBuf = Buffer.from(arrayBuffer);
 
     const magic = trailerBuf.slice(4).toString();
     if (magic !== PARQUET_MAGIC) {
@@ -162,7 +164,8 @@ export class ParquetReader {
       throw new Error(`Invalid metadata size ${metadataOffset}`);
     }
 
-    const metadataBuf = await this.file.read(metadataOffset, metadataSize);
+    const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);
+    const metadataBuf = Buffer.from(arrayBuffer2);
     // let metadata = new parquet_thrift.FileMetaData();
     // parquet_util.decodeThrift(metadata, metadataBuf);
 
@@ -244,7 +247,8 @@ export class ParquetReader {
     }
 
     dictionary = context.dictionary?.length ? context.dictionary : dictionary;
-    const pagesBuf = await this.file.read(pagesOffset, pagesSize);
+    const arrayBuffer = await this.file.read(pagesOffset, pagesSize);
+    const pagesBuf = Buffer.from(arrayBuffer);
     return await decodeDataPages(pagesBuf, {...context, dictionary});
   }
 
@@ -275,7 +279,8 @@ export class ParquetReader {
       this.file.size - dictionaryPageOffset,
       this.props.defaultDictionarySize
     );
-    const pagesBuf = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);
+    const pagesBuf = Buffer.from(arrayBuffer);
 
     const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
     const decodedPage = await decodePage(cursor, context);

package/src/parquetjs/utils/file-utils.ts

@@ -1,5 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
-import {fs, stream} from '@loaders.gl/loader-utils';
+import {stream} from '@loaders.gl/loader-utils';
+import * as fs from 'fs';
 
 export function load(name: string): any {
   return (module || (global as any)).require(name);