@loaders.gl/parquet 4.2.0-alpha.6 → 4.2.0-beta.1

package/src/parquet-loader.ts

@@ -46,11 +46,10 @@ export type ParquetLoaderOptions = LoaderOptions & {
 /**
  * ParquetJS table loader
  */
-export const ParquetWorkerLoader: Loader<
-  ObjectRowTable,
-  ObjectRowTableBatch,
-  ParquetLoaderOptions
-> = {
+export const ParquetWorkerLoader = {
+  dataType: null as unknown as ObjectRowTable,
+  batchType: null as unknown as ObjectRowTableBatch,
+
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -70,69 +69,75 @@ export const ParquetWorkerLoader: Loader<
       preserveBinary: false
     }
   }
-};
+} as const satisfies Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions>;
 
 /** ParquetJS table loader */
-export const ParquetLoader: LoaderWithParser<
-  ObjectRowTable | GeoJSONTable,
-  ObjectRowTableBatch | GeoJSONTableBatch,
-  ParquetLoaderOptions
-> = {
+export const ParquetLoader = {
   ...ParquetWorkerLoader,
+
+  dataType: null as unknown as ObjectRowTable | GeoJSONTable,
+  batchType: null as unknown as ObjectRowTableBatch | GeoJSONTableBatch,
+
   parse: (arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) =>
     parseParquetFile(new BlobFile(arrayBuffer), options),
 
   parseFile: parseParquetFile,
   parseFileInBatches: parseParquetFileInBatches
-};
+} as const satisfies LoaderWithParser<
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
+  ParquetLoaderOptions
+>;
 
 // Defeat tree shaking
 // @ts-ignore
 ParquetLoader.Buffer = Buffer;
 
-export const GeoParquetWorkerLoader: Loader<GeoJSONTable, GeoJSONTableBatch, ParquetLoaderOptions> =
-  {
-    name: 'Apache Parquet',
-    id: 'parquet',
-    module: 'parquet',
-    version: VERSION,
-    worker: true,
-    category: 'table',
-    extensions: ['parquet'],
-    mimeTypes: ['application/octet-stream'],
-    binary: true,
-    tests: ['PAR1', 'PARE'],
-    options: {
-      parquet: {
-        shape: 'geojson-table',
-        columnList: [],
-        geoparquet: true,
-        url: undefined,
-        preserveBinary: false
-      }
+export const GeoParquetWorkerLoader = {
+  dataType: null as unknown as GeoJSONTable,
+  batchType: null as unknown as GeoJSONTableBatch,
+
+  name: 'Apache Parquet',
+  id: 'parquet',
+  module: 'parquet',
+  version: VERSION,
+  worker: true,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: {
+    parquet: {
+      shape: 'geojson-table',
+      columnList: [],
+      geoparquet: true,
+      url: undefined,
+      preserveBinary: false
     }
-  };
+  }
+} as const satisfies Loader<GeoJSONTable, GeoJSONTableBatch, ParquetLoaderOptions>;
 
 /** ParquetJS table loader */
-export const GeoParquetLoader: LoaderWithParser<
-  ObjectRowTable | GeoJSONTable,
-  ObjectRowTableBatch | GeoJSONTableBatch,
-  ParquetLoaderOptions
-> = {
+export const GeoParquetLoader = {
   ...GeoParquetWorkerLoader,
+
   parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
     return parseGeoParquetFile(new BlobFile(arrayBuffer), options);
   },
   parseFile: parseGeoParquetFile,
   parseFileInBatches: parseGeoParquetFileInBatches
-};
+} as const satisfies LoaderWithParser<
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
+  ParquetLoaderOptions
+>;
 
 /** @deprecated Test to see if we can improve perf of parquetjs loader */
-export const ParquetColumnarWorkerLoader: Loader<
-  ColumnarTable,
-  ColumnarTableBatch,
-  ParquetLoaderOptions
-> = {
+export const ParquetColumnarWorkerLoader = {
+  dataType: null as any as ColumnarTable,
+  batchType: null as any as ColumnarTableBatch,
+
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -144,18 +149,14 @@ export const ParquetColumnarWorkerLoader: Loader<
   binary: true,
   tests: ['PAR1', 'PARE'],
   options: ParquetLoader.options
-};
+} as const satisfies Loader<ColumnarTable, ColumnarTableBatch, ParquetLoaderOptions>;
 
 /** @deprecated Test to see if we can improve perf of parquetjs loader */
-export const ParquetColumnarLoader: LoaderWithParser<
-  ColumnarTable,
-  ColumnarTableBatch,
-  ParquetLoaderOptions
-> = {
+export const ParquetColumnarLoader = {
   ...ParquetColumnarWorkerLoader,
   parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
     return parseParquetFileInColumns(new BlobFile(arrayBuffer), options);
   },
   parseFile: parseParquetFileInColumns,
   parseFileInBatches: parseParquetFileInColumnarBatches
-};
+} as const satisfies LoaderWithParser<ColumnarTable, ColumnarTableBatch, ParquetLoaderOptions>;

package/src/parquet-wasm-loader.ts

@@ -17,7 +17,10 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
 };
 
 /** Parquet WASM table loader */
-export const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptions> = {
+export const ParquetWasmWorkerLoader = {
+  dataType: null as unknown as ArrowTable,
+  batchType: null as never,
+
   name: 'Apache Parquet',
   id: 'parquet-wasm',
   module: 'parquet',
@@ -34,13 +37,13 @@ export const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoade
       wasmUrl: PARQUET_WASM_URL
     }
   }
-};
+} as const satisfies Loader<ArrowTable, never, ParquetWasmLoaderOptions>;
 
 /** Parquet WASM table loader */
-export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
+export const ParquetWasmLoader = {
   ...ParquetWasmWorkerLoader,
   parse(arrayBuffer: ArrayBuffer, options?: ParquetWasmLoaderOptions) {
     options = {parquet: {...ParquetWasmLoader.options.parquet, ...options?.parquet}, ...options};
     return parseParquetWasm(arrayBuffer, options);
   }
-};
+} as const satisfies LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions>;

package/src/parquet-wasm-writer.ts

@@ -16,7 +16,7 @@ export type ParquetWriterOptions = WriterOptions & {
 };
 
 /** Parquet WASM writer */
-export const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWriterOptions> = {
+export const ParquetWasmWriter = {
   name: 'Apache Parquet',
   id: 'parquet-wasm',
   module: 'parquet',
@@ -33,4 +33,4 @@ export const ParquetWasmWriter: WriterWithEncoder<ArrowTable, never, ParquetWrit
     options = {parquet: {...ParquetWasmWriter.options.parquet, ...options?.parquet}, ...options};
     return encode(arrowTable, options);
   }
-};
+} as const satisfies WriterWithEncoder<ArrowTable, never, ParquetWriterOptions>;

package/src/parquet-writer.ts

@@ -11,7 +11,7 @@ const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
 export type ParquetWriterOptions = {};
 
-export const ParquetWriter: WriterWithEncoder<Table, TableBatch, ParquetWriterOptions> = {
+export const ParquetWriter = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -22,7 +22,7 @@ export const ParquetWriter: WriterWithEncoder<Table, TableBatch, ParquetWriterOp
   options: {},
   encode: async (data, options) => encodeSync(data, options),
   encodeSync
-};
+} as const satisfies WriterWithEncoder<Table, TableBatch, ParquetWriterOptions>;
 
 function encodeSync(data, options?: ParquetWriterOptions) {
   return new ArrayBuffer(0);

package/src/parquetjs/compression.ts

@@ -12,6 +12,7 @@ import {
   LZ4Compression,
   ZstdCompression
 } from '@loaders.gl/compression';
+import {registerJSModules} from '@loaders.gl/loader-utils';
 
 import {ParquetCompression} from './schema/declare';
 
@@ -72,9 +73,12 @@ export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression
  * Register compressions that have big external libraries
  * @param options.modules External library dependencies
  */
-export async function preloadCompressions(options?: {modules: {[key: string]: any}}) {
+export async function preloadCompressions(options?: {modules?: {[key: string]: any}}) {
+  registerJSModules(options?.modules);
   const compressions = Object.values(PARQUET_COMPRESSION_METHODS);
-  return await Promise.all(compressions.map((compression) => compression.preload()));
+  return await Promise.all(
+    compressions.map((compression) => compression.preload(options?.modules))
+  );
 }
 
 /**

package/src/parquetjs/schema/types.ts

@@ -340,7 +340,8 @@ function fromPrimitive_JSON(value: any): unknown {
 }
 
 function toPrimitive_BSON(value: any): Buffer {
-  const arrayBuffer = BSONWriter.encodeSync?.(value) as ArrayBuffer;
+  // @ts-ignore
+  const arrayBuffer: ArrayBuffer = BSONWriter.encodeSync?.(value);
   return Buffer.from(arrayBuffer);
 }