@loaders.gl/parquet 4.2.0-alpha.5 → 4.2.0-beta.1

package/src/parquetjs/compression.ts

@@ -12,6 +12,7 @@ import {
   LZ4Compression,
   ZstdCompression
 } from '@loaders.gl/compression';
+import {registerJSModules} from '@loaders.gl/loader-utils';
 
 import {ParquetCompression} from './schema/declare';
 
@@ -72,9 +73,12 @@ export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression
  * Register compressions that have big external libraries
  * @param options.modules External library dependencies
  */
-export async function preloadCompressions(options?: {modules: {[key: string]: any}}) {
+export async function preloadCompressions(options?: {modules?: {[key: string]: any}}) {
+  registerJSModules(options?.modules);
   const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-  return await Promise.all(compressions.map((compression) => compression.preload()));
+  return await Promise.all(
+    compressions.map((compression) => compression.preload(options?.modules))
+  );
 }
 
 /**

package/src/parquetjs/parser/parquet-reader.ts

@@ -36,7 +36,8 @@ export type ParquetIterationProps = {
  */
 export class ParquetReader {
   static defaultProps: Required<ParquetReaderProps> = {
-    defaultDictionarySize: 1e6,
+    // max ArrayBuffer size in js is 2Gb
+    defaultDictionarySize: 2147483648,
     preserveBinary: false
   };
 

package/src/parquetjs/schema/types.ts

@@ -340,7 +340,8 @@ function fromPrimitive_JSON(value: any): unknown {
 }
 
 function toPrimitive_BSON(value: any): Buffer {
-  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;
+  // @ts-ignore
+  const arrayBuffer: ArrayBuffer = BSONWriter.encodeSync?.(value);
   return Buffer.from(arrayBuffer);
 }
 
@@ -350,6 +351,7 @@ function fromPrimitive_BSON(value: any) {
 
 function toPrimitive_TIME_MILLIS(value: any) {
   const v = parseInt(value, 10);
+  // eslint-disable-next-line @typescript-eslint/no-loss-of-precision
   if (v < 0 || v > 0xffffffffffffffff || isNaN(v)) {
     throw new Error(`invalid value for TIME_MILLIS: ${value}`);
   }

package/src/polyfills/buffer/buffer.ts

@@ -1901,7 +1901,6 @@ function writeDouble(buf: Buffer, value, offset, littleEndian, noAssert): number
 }
 
 // CUSTOM ERRORS
-// =============
 
 // Simplified versions from Node, changed for Buffer-only usage
 const errors: Record<string, any> = {};
@@ -1993,7 +1992,6 @@ function addNumericalSeparator(val) {
 }
 
 // CHECK FUNCTIONS
-// ===============
 
 function checkBounds(buf, offset, byteLength) {
   validateNumber(offset, 'offset');
@@ -2042,7 +2040,6 @@ function boundsError(value, length, type?) {
 }
 
 // HELPER FUNCTIONS
-// ================
 
 const INVALID_BASE64_RE = /[^+/0-9A-Za-z-_]/g;