@loaders.gl/parquet 4.4.2 → 5.0.0-alpha.1

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -37,12 +37,14 @@ import {
 } from '../parquet-thrift/index';
 import {osopen, oswrite, osclose} from '../utils/file-utils';
 import {getBitWidth, serializeThrift} from '../utils/read-utils';
-import Int64 from 'node-int64';
+import {concatUint8Arrays, encodeUtf8, writeUInt32LE} from '../utils/binary-utils';
+import {CompactInt64} from '../utils/uint8-array-compact-protocol';
 
 /**
  * Parquet File Magic String
  */
 const PARQUET_MAGIC = 'PAR1';
+const PARQUET_MAGIC_BYTES = encodeUtf8(PARQUET_MAGIC);
 
 /**
  * Parquet File Format Version
@@ -82,7 +84,7 @@ export interface ParquetEncoderOptions {
  * are written.
  */
 // eslint-disable-next-line @typescript-eslint/no-unused-vars
-export class ParquetEncoder<T> {
+export class ParquetEncoder<_T> {
   /**
    * Convenience method to create a new buffered parquet writer that writes to
    * the specified file
@@ -126,8 +128,7 @@ export class ParquetEncoder<T> {
   ) {
     this.schema = schema;
     this.envelopeWriter = envelopeWriter;
-    // @ts-ignore Row buffer typings...
-    this.rowBuffer = {};
+    this.rowBuffer = schema.rowGroup();
     this.rowGroupSize = opts.rowGroupSize || PARQUET_DEFAULT_ROW_GROUP_SIZE;
     this.closed = false;
     this.userMetadata = {};
@@ -156,8 +157,8 @@ export class ParquetEncoder<T> {
     }
     Shred.shredRecord(this.schema, row, this.rowBuffer);
     if (this.rowBuffer.rowCount >= this.rowGroupSize) {
-      // @ts-ignore
-      this.rowBuffer = {};
+      await this.envelopeWriter.writeRowGroup(this.rowBuffer);
+      this.rowBuffer = this.schema.rowGroup();
     }
   }
 
@@ -175,8 +176,8 @@ export class ParquetEncoder<T> {
     this.closed = true;
 
     if (this.rowBuffer.rowCount > 0 || this.rowBuffer.rowCount >= this.rowGroupSize) {
-      // @ts-ignore
-      this.rowBuffer = {};
+      await this.envelopeWriter.writeRowGroup(this.rowBuffer);
+      this.rowBuffer = this.schema.rowGroup();
     }
 
     await this.envelopeWriter.writeFooter(this.userMetadata);
@@ -236,7 +237,7 @@ export class ParquetEnvelopeWriter {
   }
 
   public schema: ParquetSchema;
-  public write: (buf: Buffer) => Promise<void>;
+  public write: (buf: Uint8Array) => Promise<void>;
   public close: () => Promise<void>;
   public offset: number;
   public rowCount: number;
@@ -246,7 +247,7 @@ export class ParquetEnvelopeWriter {
 
   constructor(
     schema: ParquetSchema,
-    writeFn: (buf: Buffer) => Promise<void>,
+    writeFn: (buf: Uint8Array) => Promise<void>,
     closeFn: () => Promise<void>,
     fileOffset: number,
     opts: ParquetEncoderOptions
@@ -261,7 +262,7 @@ export class ParquetEnvelopeWriter {
     this.useDataPageV2 = 'useDataPageV2' in opts ? Boolean(opts.useDataPageV2) : false;
   }
 
-  writeSection(buf: Buffer): Promise<void> {
+  writeSection(buf: Uint8Array): Promise<void> {
     this.offset += buf.length;
     return this.write(buf);
   }
@@ -270,7 +271,7 @@ export class ParquetEnvelopeWriter {
    * Encode the parquet file header
    */
   writeHeader(): Promise<void> {
-    return this.writeSection(Buffer.from(PARQUET_MAGIC));
+    return this.writeSection(PARQUET_MAGIC_BYTES);
   }
 
   /**
@@ -373,10 +374,10 @@ async function encodeDataPage(
 ): Promise<{
   header: PageHeader;
   headerSize: number;
-  page: Buffer;
+  page: Uint8Array;
 }> {
   /* encode repetition and definition levels */
-  let rLevelsBuf: Buffer = Buffer.alloc(0);
+  let rLevelsBuf: Uint8Array = new Uint8Array(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax)
@@ -384,7 +385,7 @@ async function encodeDataPage(
     });
   }
 
-  let dLevelsBuf: Buffer = Buffer.alloc(0);
+  let dLevelsBuf: Uint8Array = new Uint8Array(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax)
@@ -398,7 +399,7 @@ async function encodeDataPage(
     bitWidth: column.typeLength
   });
 
-  const dataBuf = Buffer.concat([rLevelsBuf, dLevelsBuf, valuesBuf] as Uint8Array[]);
+  const dataBuf = concatUint8Arrays([rLevelsBuf, dLevelsBuf, valuesBuf]);
 
   // compression = column.compression === 'UNCOMPRESSED' ? (compression || 'UNCOMPRESSED') : column.compression;
   const compressedBuf = await Compression.deflate(column.compression!, dataBuf);
@@ -418,7 +419,7 @@ async function encodeDataPage(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, compressedBuf] as Uint8Array[]);
+  const page = concatUint8Arrays([headerBuf, compressedBuf]);
 
   return {header, headerSize: headerBuf.length, page};
 }
@@ -433,7 +434,7 @@ async function encodeDataPageV2(
 ): Promise<{
   header: PageHeader;
   headerSize: number;
-  page: Buffer;
+  page: Uint8Array;
 }> {
   /* encode values */
   const valuesBuf = encodeValues(column.primitiveType!, column.encoding!, data.values, {
@@ -445,7 +446,7 @@ async function encodeDataPageV2(
   const compressedBuf = await Compression.deflate(column.compression!, valuesBuf);
 
   /* encode repetition and definition levels */
-  let rLevelsBuf: Buffer = Buffer.alloc(0);
+  let rLevelsBuf: Uint8Array = new Uint8Array(0);
   if (column.rLevelMax > 0) {
     rLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.rlevels, {
       bitWidth: getBitWidth(column.rLevelMax),
@@ -453,7 +454,7 @@ async function encodeDataPageV2(
     });
   }
 
-  let dLevelsBuf: Buffer = Buffer.alloc(0);
+  let dLevelsBuf: Uint8Array = new Uint8Array(0);
   if (column.dLevelMax > 0) {
     dLevelsBuf = encodeValues(PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING, data.dlevels, {
       bitWidth: getBitWidth(column.dLevelMax),
@@ -479,7 +480,7 @@ async function encodeDataPageV2(
 
   /* concat page header, repetition and definition levels and values */
   const headerBuf = serializeThrift(header);
-  const page = Buffer.concat([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf] as Uint8Array[]);
+  const page = concatUint8Arrays([headerBuf, rLevelsBuf, dLevelsBuf, compressedBuf]);
   return {header, headerSize: headerBuf.length, page};
 }
 
@@ -492,15 +493,15 @@ async function encodeColumnChunk(
   offset: number,
   opts: ParquetEncoderOptions
 ): Promise<{
-  body: Buffer;
+  body: Uint8Array;
   metadata: ColumnMetaData;
   metadataOffset: number;
 }> {
   const data = buffer.columnData[column.path.join()];
   const baseOffset = (opts.baseOffset || 0) + offset;
   /* encode data page(s) */
-  // const pages: Buffer[] = [];
-  let pageBuf: Buffer;
+  // const pages: Uint8Array[] = [];
+  let pageBuf: Uint8Array;
   // tslint:disable-next-line:variable-name
   let total_uncompressed_size = 0;
   // tslint:disable-next-line:variable-name
@@ -515,17 +516,17 @@ async function encodeColumnChunk(
     total_compressed_size += result.header.compressed_page_size + result.headerSize;
   }
 
-  // const pagesBuf = Buffer.concat(pages);
+  // const pagesBuf = concatUint8Arrays(pages);
   // const compression = column.compression === 'UNCOMPRESSED' ? (opts.compression || 'UNCOMPRESSED') : column.compression;
 
   /* prepare metadata header */
   const metadata = new ColumnMetaData({
     path_in_schema: column.path,
-    num_values: data.count,
-    data_page_offset: baseOffset,
+    num_values: int64(data.count),
+    data_page_offset: int64(baseOffset),
     encodings: [],
-    total_uncompressed_size, //  : pagesBuf.length,
-    total_compressed_size,
+    total_uncompressed_size: int64(total_uncompressed_size), //  : pagesBuf.length,
+    total_compressed_size: int64(total_compressed_size),
     type: Type[column.primitiveType!],
     codec: CompressionCodec[column.compression!]
   });
@@ -536,7 +537,7 @@ async function encodeColumnChunk(
 
   /* concat metadata header and data pages */
   const metadataOffset = baseOffset + pageBuf.length;
-  const body = Buffer.concat([pageBuf, serializeThrift(metadata)] as Uint8Array[]);
+  const body = concatUint8Arrays([pageBuf, serializeThrift(metadata)]);
   return {body, metadata, metadataOffset};
 }
 
@@ -548,16 +549,16 @@ async function encodeRowGroup(
   data: ParquetRowGroup,
   opts: ParquetEncoderOptions
 ): Promise<{
-  body: Buffer;
+  body: Uint8Array;
   metadata: RowGroup;
 }> {
   const metadata = new RowGroup({
-    num_rows: data.rowCount,
+    num_rows: int64(data.rowCount),
     columns: [],
-    total_byte_size: 0
+    total_byte_size: int64(0)
   });
 
-  let body = Buffer.alloc(0);
+  let body: Uint8Array = new Uint8Array(0);
   for (const field of schema.fieldList) {
     if (field.isNested) {
       continue; // eslint-disable-line no-continue
@@ -566,14 +567,14 @@ async function encodeRowGroup(
     const cchunkData = await encodeColumnChunk(field, data, body.length, opts);
 
     const cchunk = new ColumnChunk({
-      file_offset: cchunkData.metadataOffset,
+      file_offset: int64(cchunkData.metadataOffset),
       meta_data: cchunkData.metadata
     });
 
     metadata.columns.push(cchunk);
-    metadata.total_byte_size = new Int64(Number(metadata.total_byte_size) + cchunkData.body.length);
+    metadata.total_byte_size = int64(Number(metadata.total_byte_size) + cchunkData.body.length);
 
-    body = Buffer.concat([body, cchunkData.body] as Uint8Array[]);
+    body = concatUint8Arrays([body, cchunkData.body]);
   }
 
   return {body, metadata};
@@ -587,11 +588,11 @@ function encodeFooter(
   rowCount: number,
   rowGroups: RowGroup[],
   userMetadata: Record<string, string>
-): Buffer {
+): Uint8Array {
   const metadata = new FileMetaData({
     version: PARQUET_VERSION,
     created_by: 'parquets',
-    num_rows: rowCount,
+    num_rows: int64(rowCount),
     row_groups: rowGroups,
     schema: [],
     key_value_metadata: []
@@ -636,10 +637,15 @@ function encodeFooter(
   }
 
   const metadataEncoded = serializeThrift(metadata);
-  const footerEncoded = Buffer.alloc(metadataEncoded.length + 8);
+  const footerEncoded = new Uint8Array(metadataEncoded.length + 8);
 
-  metadataEncoded.copy(footerEncoded as Uint8Array);
-  footerEncoded.writeUInt32LE(metadataEncoded.length, metadataEncoded.length);
-  footerEncoded.write(PARQUET_MAGIC, metadataEncoded.length + 4);
+  footerEncoded.set(metadataEncoded);
+  writeUInt32LE(footerEncoded, metadataEncoded.length, metadataEncoded.length);
+  footerEncoded.set(PARQUET_MAGIC_BYTES, metadataEncoded.length + 4);
   return footerEncoded;
 }
+
+/** Wrap a writer metadata number in the thrift int64 compatibility object. */
+function int64(value: number): any {
+  return new CompactInt64(BigInt(value));
+}

package/src/parquetjs/parquet-thrift/ColumnIndex.ts

@@ -60,7 +60,7 @@ export class ColumnIndex {
       );
     }
     if (args != null && args.null_counts != null) {
-      this.null_counts = args.null_counts.map((c) => new Int64(+c));
+      this.null_counts = args.null_counts.map(c => new Int64(+c));
     }
   }
   public write(output: thrift.TProtocol): void {

package/src/parquetjs/parquet-thrift/index.ts

@@ -10,8 +10,6 @@
  * Autogenerated by @creditkarma/thrift-typescript v3.7.2
  * DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
  */
-export {Buffer} from '../../polyfills/buffer/install-buffer-polyfill';
-
 export {TBufferedTransport, TCompactProtocol, TFramedTransport} from 'thrift';
 
 export * from './Type';

package/src/parquetjs/parser/decoders.ts

@@ -35,7 +35,7 @@ import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils'
  * @returns parquet data page data
  */
 export async function decodeDataPages(
-  buffer: Buffer,
+  buffer: Uint8Array,
   context: ParquetReaderContext
 ): Promise<ParquetColumnChunk> {
   const cursor: CursorBuffer = {
@@ -76,7 +76,7 @@ export async function decodeDataPages(
     // of 'PLAIN' encoding because all values are already in place
     if (dictionary.length && valueEncoding !== 'PLAIN') {
       // eslint-disable-next-line no-loop-func
-      page.values = page.values.map((value) => dictionary[value]);
+      page.values = page.values.map(value => dictionary[value]);
     }
 
     for (let index = 0; index < page.rlevels.length; index++) {
@@ -187,7 +187,7 @@ export function decodeSchema(
       const type = getThriftEnum(Type, schemaElement.type!);
       let logicalType = type;
 
-      if (schemaElement.converted_type) {
+      if (schemaElement.converted_type !== undefined && schemaElement.converted_type !== null) {
         logicalType = getThriftEnum(ConvertedType, schemaElement.converted_type);
       }
 
@@ -435,7 +435,7 @@ async function decodeDictionaryPage(
   if (context.compression !== 'UNCOMPRESSED') {
     const valuesBuf = await decompress(
       context.compression,
-      dictCursor.buffer.slice(dictCursor.offset, cursorEnd),
+      dictCursor.buffer.subarray(dictCursor.offset),
       pageHeader.uncompressed_page_size
     );
 
@@ -459,23 +459,5 @@ async function decodeDictionaryPage(
     context as ParquetCodecOptions
   );
 
-  // Makes it look a little easier
-  let values: any[];
-  if (context?.preserveBinary) {
-    values = decodedDictionaryValues.map((d) => preserveBinary(d));
-  } else {
-    values = decodedDictionaryValues.map((d) => d.toString());
-  }
-  return values;
-}
-
-function preserveBinary(d: any): ArrayBuffer | ArrayBufferView | string {
-  if (ArrayBuffer.isView(d)) {
-    return d;
-  }
-  // Convert to ArrayBuffer
-  if (Buffer.isBuffer(d)) {
-    return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
-  }
-  return d.toString();
+  return decodedDictionaryValues;
 }

package/src/parquetjs/parser/parquet-reader.ts

@@ -20,6 +20,7 @@ import {
   ParquetReaderContext
 } from '../schema/declare';
 import {decodeFileMetadata, getThriftEnum, fieldIndexOf} from '../utils/read-utils';
+import {decodeString, readUInt32LE, toUint8Array} from '../utils/binary-utils';
 
 export type ParquetReaderProps = {
   defaultDictionarySize?: number;
@@ -82,9 +83,7 @@ export class ParquetReader {
   /** Iterate over the raw row groups */
   async *rowGroupIterator(props?: ParquetIterationProps) {
     // Ensure strings are nested in arrays
-    const columnList: string[][] = (props?.columnList || []).map((x) =>
-      Array.isArray(x) ? x : [x]
-    );
+    const columnList: string[][] = (props?.columnList || []).map(x => (Array.isArray(x) ? x : [x]));
 
     const metadata = await this.getFileMetadata();
     const schema = await this.getSchema();
@@ -140,8 +139,7 @@ export class ParquetReader {
   /** Metadata is stored in the footer */
   async readHeader(): Promise<void> {
     const arrayBuffer = await this.file.read(0, PARQUET_MAGIC.length);
-    const buffer = Buffer.from(arrayBuffer);
-    const magic = buffer.toString();
+    const magic = decodeString(toUint8Array(arrayBuffer));
     switch (magic) {
       case PARQUET_MAGIC:
         break;
@@ -156,21 +154,21 @@ export class ParquetReader {
   async readFooter(): Promise<FileMetaData> {
     const trailerLen = PARQUET_MAGIC.length + 4;
     const arrayBuffer = await this.file.read(this.file.size - trailerLen, trailerLen);
-    const trailerBuf = Buffer.from(arrayBuffer);
+    const trailer = toUint8Array(arrayBuffer);
 
-    const magic = trailerBuf.slice(4).toString();
+    const magic = decodeString(trailer, 4);
     if (magic !== PARQUET_MAGIC) {
       throw new Error(`Not a valid parquet file (magic="${magic})`);
     }
 
-    const metadataSize = trailerBuf.readUInt32LE(0);
+    const metadataSize = readUInt32LE(trailer, 0);
     const metadataOffset = this.file.size - metadataSize - trailerLen;
     if (metadataOffset < PARQUET_MAGIC.length) {
       throw new Error(`Invalid metadata size ${metadataOffset}`);
     }
 
     const arrayBuffer2 = await this.file.read(metadataOffset, metadataSize);
-    const metadataBuf = Buffer.from(arrayBuffer2);
+    const metadataBuf = toUint8Array(arrayBuffer2);
     // let metadata = new parquet_thrift.FileMetaData();
     // parquet_util.decodeThrift(metadata, metadataBuf);
 
@@ -253,7 +251,7 @@ export class ParquetReader {
 
     dictionary = context.dictionary?.length ? context.dictionary : dictionary;
     const arrayBuffer = await this.file.read(pagesOffset, pagesSize);
-    const pagesBuf = Buffer.from(arrayBuffer);
+    const pagesBuf = toUint8Array(arrayBuffer);
     return await decodeDataPages(pagesBuf, {...context, dictionary});
   }
 
@@ -268,7 +266,7 @@ export class ParquetReader {
     dictionaryPageOffset: number,
     context: ParquetReaderContext,
     pagesOffset: number
-  ): Promise<string[]> {
+  ): Promise<any[]> {
     if (dictionaryPageOffset === 0) {
       // dictionarySize = Math.min(this.fileSize - pagesOffset, this.defaultDictionarySize);
       // pagesBuf = await this.read(pagesOffset, dictionarySize);
@@ -285,7 +283,7 @@ export class ParquetReader {
       this.props.defaultDictionarySize
     );
     const arrayBuffer = await this.file.read(dictionaryPageOffset, dictionarySize);
-    const pagesBuf = Buffer.from(arrayBuffer);
+    const pagesBuf = toUint8Array(arrayBuffer);
 
     const cursor = {buffer: pagesBuf, offset: 0, size: pagesBuf.length};
     const decodedPage = await decodePage(cursor, context);

package/src/parquetjs/schema/declare.ts

@@ -66,7 +66,7 @@ export type OriginalType =
   | 'BSON' // 20
   | 'INTERVAL'; // 21
 
-export type ParquetDictionary = string[];
+export type ParquetDictionary = any[];
 
 export interface SchemaDefinition {
   [string: string]: FieldDefinition;