@loaders.gl/parquet 5.0.0-alpha.2 → 5.0.0-alpha.3

package/src/parquetjs/compression.ts

@@ -5,12 +5,22 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license
 // Forked from https://github.com/ironSource/parquetjs under MIT license
 
-import type {Compression} from '@loaders.gl/compression';
 import {
-  decompressWithNativeDecompressionStream,
-  type NativeDecompressionFormat
-} from '@loaders.gl/compression/native-decompression';
-import {getJSModuleOrNull, registerJSModules} from '@loaders.gl/loader-utils';
+  BrotliCompressor,
+  BrotliDecompressor,
+  GZipCompressor,
+  GZipDecompressor,
+  LZ4Compressor,
+  LZ4Decompressor,
+  SnappyCompressor,
+  SnappyDecompressor,
+  ZstdCompressor,
+  ZstdDecompressor,
+  Decompressor,
+  type Compressor
+} from '@loaders.gl/compression';
+import {SnappyHysnappyDecompressor} from '@loaders.gl/compression/snappy-decompressor-hysnappy';
+import {registerJSModules} from '@loaders.gl/loader-utils';
 
 import {ParquetCompression} from './schema/declare';
 import {toArrayBuffer, toUint8Array} from './utils/binary-utils';
@@ -29,17 +39,40 @@ export const PARQUET_COMPRESSION_METHODS: Partial<Record<ParquetCompression, tru
   ZSTD: true
 };
 
-/** Native formats available to asynchronous Parquet page decompression. */
-const PARQUET_NATIVE_DECOMPRESSION_FORMATS: Partial<
-  Record<ParquetCompression, NativeDecompressionFormat>
-> = {
-  GZIP: 'gzip',
-  BROTLI: 'brotli',
-  ZSTD: 'zstd'
-};
+/** Reader-scoped function that decompresses one independently encoded Parquet page. */
+export type ParquetPageDecompressor = (value: Uint8Array, size: number) => Promise<Uint8Array>;
+
+/** Fast Snappy decoder that permanently falls back to snappyjs when WASM is unavailable. */
+class ParquetSnappyDecompressor extends Decompressor {
+  /** Compression format name. */
+  readonly name = 'snappy';
+  /** Snappy does not have a standard standalone file extension. */
+  readonly extensions: string[] = [];
+  /** Snappy does not have a standard HTTP content encoding. */
+  readonly contentEncodings: string[] = [];
+  /** snappyjs keeps this composite decoder available without WebAssembly. */
+  readonly isSupported = true;
+  /** Preferred compact WASM decoder for page-sized frames. */
+  private readonly hysnappy = new SnappyHysnappyDecompressor();
+  /** Always-supported JavaScript decoder. */
+  private readonly snappyjs = new SnappyDecompressor();
+  /** Whether this instance has selected the JavaScript fallback. */
+  private useSnappyjs = !this.hysnappy.isSupported;
 
-/** Lazily constructed codec-backed compression implementations. */
-const compressionPromises: Partial<Record<ParquetCompression, Promise<Compression>>> = {};
+  /** Decodes with hysnappy, retaining snappyjs after the first WASM setup or runtime failure. */
+  override async decompress(input: ArrayBuffer, size?: number): Promise<ArrayBuffer> {
+    if (!this.useSnappyjs) {
+      try {
+        return await this.hysnappy.decompress(input, size);
+      } catch {
+        // CSP can reject WebAssembly compilation even when the WebAssembly global exists. Once
+        // that happens, avoid paying the same rejected initialization cost for every Parquet page.
+        this.useSnappyjs = true;
+      }
+    }
+    return await this.snappyjs.decompress(input, size);
+  }
+}
 
 /**
  * Registers optional codec modules without eagerly loading codec-backed implementations.
@@ -61,7 +94,7 @@ export async function deflate(method: ParquetCompression, value: Uint8Array): Pr
     return value;
   }
 
-  const compression = await getParquetCompression(method);
+  const compression = await getParquetCompressor(method);
   const inputArrayBuffer = toArrayBuffer(value);
   const compressedArrayBuffer = await compression.compress(inputArrayBuffer);
   return toUint8Array(compressedArrayBuffer);
@@ -83,79 +116,72 @@ export async function decompress(
     return toUint8Array(inputArrayBuffer);
   }
 
-  const nativeFormat = PARQUET_NATIVE_DECOMPRESSION_FORMATS[method];
-  if (nativeFormat && shouldUseNativeDecompressionStream(method)) {
-    const nativeOutput = await decompressWithNativeDecompressionStream(
-      inputArrayBuffer,
-      nativeFormat
-    );
-    if (nativeOutput) {
-      return toUint8Array(nativeOutput);
-    }
-  }
-
-  const compression = await getParquetCompression(method);
+  const compression = await getParquetDecompressor(method);
   const compressedArrayBuffer = await compression.decompress(inputArrayBuffer, size);
   return toUint8Array(compressedArrayBuffer);
 }
 
 /**
- * Returns whether a native stream can take precedence for one Parquet compression method.
+ * Creates a reusable decoder for the independently compressed pages in one Parquet reader.
  *
- * @param method Parquet compression method.
- * @returns Whether no explicitly registered codec should take precedence.
+ * Reusing the lazy codec preserves its selected backend and avoids repeating dynamic import and
+ * preload work for every page while keeping module injection scoped to a reader invocation.
  */
-function shouldUseNativeDecompressionStream(method: ParquetCompression): boolean {
-  if (method === 'BROTLI') {
-    return !getJSModuleOrNull('brotli');
-  }
-  if (method === 'ZSTD') {
-    return !getJSModuleOrNull('zstd-codec');
-  }
-  return true;
+export function createParquetPageDecompressor(method: ParquetCompression): ParquetPageDecompressor {
+  const decompressor = method === 'UNCOMPRESSED' ? null : createParquetDecompressor(method);
+  return async (value: Uint8Array, size: number): Promise<Uint8Array> => {
+    const inputArrayBuffer = toArrayBuffer(value);
+    if (!decompressor) {
+      return toUint8Array(inputArrayBuffer);
+    }
+    const decompressedArrayBuffer = await decompressor.decompress(inputArrayBuffer, size);
+    return toUint8Array(decompressedArrayBuffer);
+  };
 }
 
-/**
- * Loads one codec-backed implementation only after native decompression is unavailable.
- *
- * @param method Parquet compression method.
- * @returns Codec-backed compression implementation.
- */
-async function getParquetCompression(method: ParquetCompression): Promise<Compression> {
-  compressionPromises[method] ||= createParquetCompression(method);
-  return await compressionPromises[method];
+/** Returns a new lazily selecting compressor for one Parquet method. */
+async function getParquetCompressor(method: ParquetCompression): Promise<Compressor> {
+  return createParquetCompressor(method);
 }
 
-/**
- * Creates one lazily loaded codec-backed Parquet compression implementation.
- *
- * @param method Parquet compression method.
- * @returns Codec-backed compression implementation.
- */
-async function createParquetCompression(method: ParquetCompression): Promise<Compression> {
+/** Returns a new lazily selecting decompressor for one Parquet method. */
+async function getParquetDecompressor(method: ParquetCompression): Promise<Decompressor> {
+  return createParquetDecompressor(method);
+}
+
+/** Creates the root-level default compressor for one Parquet method. */
+function createParquetCompressor(method: ParquetCompression): Compressor {
   switch (method) {
-    case 'GZIP': {
-      const {GZipCompression} = await import('@loaders.gl/compression/gzip-compression');
-      return new GZipCompression();
-    }
-    case 'SNAPPY': {
-      const {SnappyCompression} = await import('@loaders.gl/compression/snappy-compression');
-      return new SnappyCompression();
-    }
-    case 'BROTLI': {
-      const {BrotliCompression} = await import('@loaders.gl/compression/brotli-compression');
-      return new BrotliCompression();
-    }
+    case 'GZIP':
+      return new GZipCompressor();
+    case 'SNAPPY':
+      return new SnappyCompressor();
+    case 'BROTLI':
+      return new BrotliCompressor();
     case 'LZ4':
-    case 'LZ4_RAW': {
-      const {LZ4Compression} = await import('@loaders.gl/compression/lz4-compression');
-      const lz4js = getJSModuleOrNull('lz4js') || (await import('lz4js')).default;
-      return new LZ4Compression({modules: {lz4js}});
-    }
-    case 'ZSTD': {
-      const {ZstdCompression} = await import('@loaders.gl/compression/zstd-compression');
-      return new ZstdCompression();
-    }
+    case 'LZ4_RAW':
+      return new LZ4Compressor();
+    case 'ZSTD':
+      return new ZstdCompressor();
+    default:
+      throw new Error(`parquet: invalid compression method: ${method}`);
+  }
+}
+
+/** Creates the root-level default decompressor for one Parquet method. */
+function createParquetDecompressor(method: ParquetCompression): Decompressor {
+  switch (method) {
+    case 'GZIP':
+      return new GZipDecompressor();
+    case 'SNAPPY':
+      return new ParquetSnappyDecompressor();
+    case 'BROTLI':
+      return new BrotliDecompressor();
+    case 'LZ4':
+    case 'LZ4_RAW':
+      return new LZ4Decompressor();
+    case 'ZSTD':
+      return new ZstdDecompressor();
     default:
       throw new Error(`parquet: invalid compression method: ${method}`);
   }

package/src/parquetjs/encoder/dictionary-planner.ts

@@ -32,7 +32,12 @@ export function planDictionary(
       `Parquet dictionary page size limit must be a positive integer, received ${dictionaryPageSizeLimit}`
     );
   }
-  if (policy === false || values.length === 0 || isDictionaryEncoding(column.encoding!)) {
+  if (policy === false || values.length === 0) {
+    if (policy === false && isDictionaryEncoding(column.encoding!)) {
+      throw new Error(
+        `Parquet column ${column.path.join('.')} requests ${column.encoding} but dictionary encoding is disabled`
+      );
+    }
     return undefined;
   }
 
@@ -58,7 +63,7 @@ export function planDictionary(
     return undefined;
   }
   const bitWidth = dictionaryValues.length <= 1 ? 0 : Math.ceil(Math.log2(dictionaryValues.length));
-  if (policy === 'auto') {
+  if (policy === 'auto' && !isDictionaryEncoding(column.encoding!)) {
     const encodedIndices = PARQUET_CODECS.RLE_DICTIONARY.encodeValues(
       column.primitiveType!,
       indices,