@loaders.gl/parquet 4.0.0-alpha.5 → 4.0.0-alpha.7

package/src/parquet-loader.ts

@@ -1,4 +1,11 @@
+// loaders.gl, MIT license
 import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ColumnarTable,
+  ColumnarTableBatch
+} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -8,18 +15,22 @@ export type ParquetLoaderOptions = LoaderOptions & {
   parquet?: {
     type?: 'object-row-table';
     url?: string;
+    columnList?: string[] | string[][];
+    geoparquet?: boolean;
   };
 };
 
 const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
   parquet: {
     type: 'object-row-table',
-    url: undefined
+    url: undefined,
+    columnList: [],
+    geoparquet: true
   }
 };
 
 /** ParquetJS table loader */
-export const ParquetLoader = {
+export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -33,4 +44,20 @@ export const ParquetLoader = {
   options: DEFAULT_PARQUET_LOADER_OPTIONS
 };
 
-export const _typecheckParquetLoader: Loader = ParquetLoader;
+export const ParqueColumnnartLoader: Loader<
+  ColumnarTable,
+  ColumnarTableBatch,
+  ParquetLoaderOptions
+> = {
+  name: 'Apache Parquet',
+  id: 'parquet',
+  module: 'parquet',
+  version: VERSION,
+  worker: true,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: DEFAULT_PARQUET_LOADER_OPTIONS
+};

package/src/parquet-wasm-loader.ts

@@ -0,0 +1,36 @@
+import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type ParquetLoaderOptions = LoaderOptions & {
+  parquet?: {
+    type?: 'arrow-table';
+    wasmUrl?: string;
+  };
+};
+
+const DEFAULT_PARQUET_LOADER_OPTIONS: ParquetLoaderOptions = {
+  parquet: {
+    type: 'arrow-table',
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+
+/** ParquetJS table loader */
+export const ParquetWasmLoader = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  worker: false,
+  category: 'table',
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  binary: true,
+  tests: ['PAR1', 'PARE'],
+  options: DEFAULT_PARQUET_LOADER_OPTIONS
+};
+
+export const _typecheckParquetLoader: Loader = ParquetWasmLoader;

package/src/parquet-wasm-writer.ts

@@ -0,0 +1,24 @@
+import type {Writer} from '@loaders.gl/loader-utils';
+import {encode, ParquetWriterOptions} from './lib/wasm/encode-parquet-wasm';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+const DEFAULT_PARQUET_WRITER_OPTIONS: ParquetWriterOptions = {
+  parquet: {
+    wasmUrl: 'https://unpkg.com/parquet-wasm@0.3.1/esm2/arrow1_bg.wasm'
+  }
+};
+
+export const ParquetWasmWriter: Writer = {
+  name: 'Apache Parquet',
+  id: 'parquet-wasm',
+  module: 'parquet',
+  version: VERSION,
+  extensions: ['parquet'],
+  mimeTypes: ['application/octet-stream'],
+  encode,
+  binary: true,
+  options: DEFAULT_PARQUET_WRITER_OPTIONS
+};

package/src/parquet-writer.ts

@@ -1,4 +1,7 @@
+// loaders.gl, MIT license
+
 import type {Writer} from '@loaders.gl/loader-utils';
+import {Table, TableBatch} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -8,7 +11,7 @@ export type ParquetWriterOptions = {};
 
 const DEFAULT_PARQUET_LOADER_OPTIONS = {};
 
-export const ParquetWriter: Writer = {
+export const ParquetWriter: Writer<Table, TableBatch, ParquetWriterOptions> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',

package/src/parquetjs/compression.ts

@@ -8,20 +8,33 @@ import {
   GZipCompression,
   SnappyCompression,
   BrotliCompression,
-  LZOCompression,
+  // LZOCompression,
   LZ4Compression,
   ZstdCompression
 } from '@loaders.gl/compression';
 
 import {ParquetCompression} from './schema/declare';
-import {toArrayBuffer, toBuffer} from './utils/buffer-utils';
+
+/** We can't use loaders-util buffer handling since we are dependent on buffers even in the browser */
+function toBuffer(arrayBuffer: ArrayBuffer): Buffer {
+  return Buffer.from(arrayBuffer);
+}
+
+function toArrayBuffer(buffer: Buffer): ArrayBuffer {
+  // TODO - per docs we should just be able to call buffer.buffer, but there are issues
+  if (Buffer.isBuffer(buffer)) {
+    const typedArray = new Uint8Array(buffer.buffer, buffer.byteOffset, buffer.length);
+    return typedArray.slice().buffer;
+  }
+  return buffer;
+}
 
 // TODO switch to worker compression to avoid bundling...
 
 // import brotli from 'brotli'; - brotli has problems with decompress in browsers
 // import brotliDecompress from 'brotli/decompress';
 import lz4js from 'lz4js';
-import lzo from 'lzo';
+// import lzo from 'lzo';
 // import {ZstdCodec} from 'zstd-codec';
 
 // Inject large dependencies through Compression constructor options
@@ -33,12 +46,15 @@ const modules = {
   //     throw new Error('brotli compress');
   //   }
   // },
-  lz4js,
-  lzo
+  lz4js
+  // lzo
   // 'zstd-codec': ZstdCodec
 };
 
-// See https://github.com/apache/parquet-format/blob/master/Compression.md
+/**
+ * See https://github.com/apache/parquet-format/blob/master/Compression.md
+ */
+// @ts-expect-error
 export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression> = {
   UNCOMPRESSED: new NoCompression(),
   GZIP: new GZipCompression(),
@@ -47,7 +63,8 @@ export const PARQUET_COMPRESSION_METHODS: Record<ParquetCompression, Compression
   // TODO: Understand difference between LZ4 and LZ4_RAW
   LZ4: new LZ4Compression({modules}),
   LZ4_RAW: new LZ4Compression({modules}),
-  LZO: new LZOCompression({modules}),
+  //
+  // LZO: new LZOCompression({modules}),
   ZSTD: new ZstdCompression({modules})
 };
 

package/src/parquetjs/encoder/{writer.ts → parquet-encoder.ts}

@@ -1,14 +1,15 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
-import {Transform, Writable} from 'stream';
+import {stream} from '@loaders.gl/loader-utils';
 import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';
 import * as Compression from '../compression';
 import {
-  ParquetBuffer,
+  ParquetRowGroup,
   ParquetCodec,
-  ParquetData,
+  ParquetColumnChunk,
   ParquetField,
-  PrimitiveType
+  PrimitiveType,
+  ParquetRow
 } from '../schema/declare';
 import {ParquetSchema} from '../schema/schema';
 import * as Shred from '../schema/shred';
@@ -55,7 +56,7 @@ const PARQUET_DEFAULT_ROW_GROUP_SIZE = 4096;
 const PARQUET_RDLVL_TYPE = 'INT32';
 const PARQUET_RDLVL_ENCODING = 'RLE';
 
-export interface ParquetWriterOptions {
+export interface ParquetEncoderOptions {
   baseOffset?: number;
   rowGroupSize?: number;
   pageSize?: number;
@@ -71,12 +72,12 @@ export interface ParquetWriterOptions {
 }
 
 /**
- * Write a parquet file to an output stream. The ParquetWriter will perform
+ * Write a parquet file to an output stream. The ParquetEncoder will perform
  * buffering/batching for performance, so close() must be called after all rows
  * are written.
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export class ParquetWriter<T> {
+export class ParquetEncoder<T> {
   /**
    * Convenience method to create a new buffered parquet writer that writes to
    * the specified file
@@ -84,10 +85,10 @@ export class ParquetWriter<T> {
   static async openFile<T>(
     schema: ParquetSchema,
     path: string,
-    opts?: ParquetWriterOptions
-  ): Promise<ParquetWriter<T>> {
+    opts?: ParquetEncoderOptions
+  ): Promise<ParquetEncoder<T>> {
     const outputStream = await osopen(path, opts);
-    return ParquetWriter.openStream(schema, outputStream, opts);
+    return ParquetEncoder.openStream(schema, outputStream, opts);
   }
 
   /**
@@ -96,22 +97,16 @@ export class ParquetWriter<T> {
    */
   static async openStream<T>(
     schema: ParquetSchema,
-    outputStream: Writable,
-    opts?: ParquetWriterOptions
-  ): Promise<ParquetWriter<T>> {
-    if (!opts) {
-      // tslint:disable-next-line:no-parameter-reassignment
-      opts = {};
-    }
-
+    outputStream: stream.Writable,
+    opts: ParquetEncoderOptions = {}
+  ): Promise<ParquetEncoder<T>> {
     const envelopeWriter = await ParquetEnvelopeWriter.openStream(schema, outputStream, opts);
-
-    return new ParquetWriter(schema, envelopeWriter, opts);
+    return new ParquetEncoder(schema, envelopeWriter, opts);
   }
 
   public schema: ParquetSchema;
   public envelopeWriter: ParquetEnvelopeWriter;
-  public rowBuffer: ParquetBuffer;
+  public rowBuffer: ParquetRowGroup;
   public rowGroupSize: number;
   public closed: boolean;
   public userMetadata: Record<string, string>;
@@ -122,7 +117,7 @@ export class ParquetWriter<T> {
   constructor(
     schema: ParquetSchema,
     envelopeWriter: ParquetEnvelopeWriter,
-    opts: ParquetWriterOptions
+    opts: ParquetEncoderOptions
   ) {
     this.schema = schema;
     this.envelopeWriter = envelopeWriter;
@@ -150,7 +145,7 @@ export class ParquetWriter<T> {
    * Append a single row to the parquet file. Rows are buffered in memory until
    * rowGroupSize rows are in the buffer or close() is called
    */
-  async appendRow<T>(row: T): Promise<void> {
+  async appendRow<T extends ParquetRow>(row: T): Promise<void> {
     if (this.closed) {
       throw new Error('writer was closed');
     }
@@ -227,8 +222,8 @@ export class ParquetEnvelopeWriter {
    */
   static async openStream(
     schema: ParquetSchema,
-    outputStream: Writable,
-    opts: ParquetWriterOptions
+    outputStream: stream.Writable,
+    opts: ParquetEncoderOptions
   ): Promise<ParquetEnvelopeWriter> {
     const writeFn = oswrite.bind(undefined, outputStream);
     const closeFn = osclose.bind(undefined, outputStream);
@@ -249,7 +244,7 @@ export class ParquetEnvelopeWriter {
     writeFn: (buf: Buffer) => Promise<void>,
     closeFn: () => Promise<void>,
     fileOffset: number,
-    opts: ParquetWriterOptions
+    opts: ParquetEncoderOptions
   ) {
     this.schema = schema;
     this.write = writeFn;
@@ -277,7 +272,7 @@ export class ParquetEnvelopeWriter {
    * Encode a parquet row group. The records object should be created using the
    * shredRecord method
    */
-  async writeRowGroup(records: ParquetBuffer): Promise<void> {
+  async writeRowGroup(records: ParquetRowGroup): Promise<void> {
     const rgroup = await encodeRowGroup(this.schema, records, {
       baseOffset: this.offset,
       pageSize: this.pageSize,
@@ -314,11 +309,10 @@ export class ParquetEnvelopeWriter {
 
 /**
  * Create a parquet transform stream
- */
-export class ParquetTransformer<T> extends Transform {
-  public writer: ParquetWriter<T>;
+export class ParquetTransformer<T> extends stream.Transform {
+  public writer: ParquetEncoder<T>;
 
-  constructor(schema: ParquetSchema, opts: ParquetWriterOptions = {}) {
+  constructor(schema: ParquetSchema, opts: ParquetEncoderOptions = {}) {
     super({objectMode: true});
 
     const writeProxy = (function (t: ParquetTransformer<any>) {
@@ -327,7 +321,7 @@ export class ParquetTransformer<T> extends Transform {
       };
     })(this);
 
-    this.writer = new ParquetWriter(
+    this.writer = new ParquetEncoder(
       schema,
       new ParquetEnvelopeWriter(schema, writeProxy, async () => {}, 0, opts),
       opts
@@ -348,6 +342,7 @@ export class ParquetTransformer<T> extends Transform {
     await this.writer.close(callback);
   }
 }
+ */
 
 /**
  * Encode a consecutive array of data using one of the parquet encodings
@@ -369,7 +364,7 @@ function encodeValues(
  */
 async function encodeDataPage(
   column: ParquetField,
-  data: ParquetData
+  data: ParquetColumnChunk
 ): Promise<{
   header: PageHeader;
   headerSize: number;
@@ -428,7 +423,7 @@ async function encodeDataPage(
  */
 async function encodeDataPageV2(
   column: ParquetField,
-  data: ParquetData,
+  data: ParquetColumnChunk,
   rowCount: number
 ): Promise<{
   header: PageHeader;
@@ -488,9 +483,9 @@ async function encodeDataPageV2(
  */
 async function encodeColumnChunk(
   column: ParquetField,
-  buffer: ParquetBuffer,
+  buffer: ParquetRowGroup,
   offset: number,
-  opts: ParquetWriterOptions
+  opts: ParquetEncoderOptions
 ): Promise<{
   body: Buffer;
   metadata: ColumnMetaData;
@@ -545,8 +540,8 @@ async function encodeColumnChunk(
  */
 async function encodeRowGroup(
   schema: ParquetSchema,
-  data: ParquetBuffer,
-  opts: ParquetWriterOptions
+  data: ParquetRowGroup,
+  opts: ParquetEncoderOptions
 ): Promise<{
   body: Buffer;
   metadata: RowGroup;

package/src/parquetjs/parser/decoders.ts

@@ -1,7 +1,7 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 import {
   ParquetCodec,
-  ParquetData,
+  ParquetColumnChunk,
   ParquetOptions,
   ParquetPageData,
   ParquetType,
@@ -32,14 +32,14 @@ import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils'
 export async function decodeDataPages(
   buffer: Buffer,
   options: ParquetOptions
-): Promise<ParquetData> {
+): Promise<ParquetColumnChunk> {
   const cursor: CursorBuffer = {
     buffer,
     offset: 0,
     size: buffer.length
   };
 
-  const data: ParquetData = {
+  const data: ParquetColumnChunk = {
     rlevels: [],
     dlevels: [],
     values: [],